Re: [u-boot][PATCH v2 1/8] mtd: rawnand: omap_gpmc: Fix BCH6/16 HW based correction

Hi

On Tue, Dec 20, 2022 at 11:22 AM Roger Quadros rogerq@kernel.org wrote:

The BCH detection hardware can generate ECC bytes for multiple sectors in one go. Use that feature.

correct() only corrects one sector at a time so we need to call it repeatedly for each sector.

Signed-off-by: Roger Quadros rogerq@kernel.org Reviewed-by: Michael Trimarchi michael@amarulasolutions.com

---

drivers/mtd/nand/raw/omap_gpmc.c | 325 +++++++++++++++++++++---------- 1 file changed, 223 insertions(+), 102 deletions(-)

diff --git a/drivers/mtd/nand/raw/omap_gpmc.c b/drivers/mtd/nand/raw/omap_gpmc.c index 69fc09be097..e772a914c88 100644 --- a/drivers/mtd/nand/raw/omap_gpmc.c +++ b/drivers/mtd/nand/raw/omap_gpmc.c @@ -27,6 +27,9 @@

#define BADBLOCK_MARKER_LENGTH 2 #define SECTOR_BYTES 512 +#define ECCSIZE0_SHIFT 12 +#define ECCSIZE1_SHIFT 22 +#define ECC1RESULTSIZE 0x1 #define ECCCLEAR (0x1 << 8) #define ECCRESULTREG1 (0x1 << 0) /* 4 bit padding to make byte aligned, 56 = 52 + 4 */ @@ -186,72 +189,35 @@ static int __maybe_unused omap_correct_data(struct mtd_info *mtd, uint8_t *dat, __maybe_unused static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode) {

•   struct nand_chip        *nand   = mtd_to_nand(mtd);
  

•   struct omap_nand_info   *info   = nand_get_controller_data(nand);
  

•   struct nand_chip *nand = mtd_to_nand(mtd);
  

•   struct omap_nand_info *info = nand_get_controller_data(nand);
    unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0;
  

•   unsigned int ecc_algo = 0;
  

•   unsigned int bch_type = 0;
  

•   unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00;
  

•   u32 ecc_size_config_val = 0;
  

•   u32 ecc_config_val = 0;
  

•   int cs = info->cs;
  

•   u32 val;
  
  

•   /* configure GPMC for specific ecc-scheme */
  

•   switch (info->ecc_scheme) {
  

•   case OMAP_ECC_HAM1_CODE_SW:
  

•           return;
  

•   case OMAP_ECC_HAM1_CODE_HW:
  

•           ecc_algo = 0x0;
  

•           bch_type = 0x0;
  

•           bch_wrapmode = 0x00;
  

•           eccsize0 = 0xFF;
  

•           eccsize1 = 0xFF;
  

•   /* Clear ecc and enable bits */
  

•   writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
  

•   /* program ecc and result sizes */
  

•   val = ((((nand->ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |
  

•                   ECC1RESULTSIZE);
  

•   writel(val, &gpmc_cfg->ecc_size_config);
  

•   switch (mode) {
  

•   case NAND_ECC_READ:
  

•   case NAND_ECC_WRITE:
  

•           writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
            break;
  

•   case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
  

•   case OMAP_ECC_BCH8_CODE_HW:
  

•           ecc_algo = 0x1;
  

•           bch_type = 0x1;
  

•           if (mode == NAND_ECC_WRITE) {
  

•                   bch_wrapmode = 0x01;
  

•                   eccsize0 = 0;  /* extra bits in nibbles per sector */
  

•                   eccsize1 = 28; /* OOB bits in nibbles per sector */
  

•           } else {
  

•                   bch_wrapmode = 0x01;
  

•                   eccsize0 = 26; /* ECC bits in nibbles per sector */
  

•                   eccsize1 = 2;  /* non-ECC bits in nibbles per sector */
  

•           }
  

•           break;
  

•   case OMAP_ECC_BCH16_CODE_HW:
  

•           ecc_algo = 0x1;
  

•           bch_type = 0x2;
  

•           if (mode == NAND_ECC_WRITE) {
  

•                   bch_wrapmode = 0x01;
  

•                   eccsize0 = 0;  /* extra bits in nibbles per sector */
  

•                   eccsize1 = 52; /* OOB bits in nibbles per sector */
  

•           } else {
  

•                   bch_wrapmode = 0x01;
  

•                   eccsize0 = 52; /* ECC bits in nibbles per sector */
  

•                   eccsize1 = 0;  /* non-ECC bits in nibbles per sector */
  

•           }
  

•   case NAND_ECC_READSYN:
  

•           writel(ECCCLEAR, &gpmc_cfg->ecc_control);
            break;
    default:
  

•           return;
  

•           printf("%s: error: unrecognized Mode[%d]!\n", __func__, mode);
  

•           break;
    }
  

•   /* Clear ecc and enable bits */
  

•   writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
  

•   /* Configure ecc size for BCH */
  

•   ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12);
  

•   writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config);
  

•   /* Configure device details for BCH engine */
  

•   ecc_config_val = ((ecc_algo << 16)      | /* HAM1 | BCHx */
  

•                   (bch_type << 12)        | /* BCH4/BCH8/BCH16 */
  

•                   (bch_wrapmode << 8)     | /* wrap mode */
  

•                   (dev_width << 7)        | /* bus width */
  

•                   (0x0 << 4)              | /* number of sectors */
  

•                   (cs <<  1)              | /* ECC CS */
  

•                   (0x1));                   /* enable ECC */
  

•   writel(ecc_config_val, &gpmc_cfg->ecc_config);
  

•   /* (ECC 16 or 8 bit col) | ( CS  )  | ECC Enable */
  

•   val = (dev_width << 7) | (info->cs << 1) | (0x1);
  

•   writel(val, &gpmc_cfg->ecc_config);
  

}

/* @@ -270,6 +236,124 @@ static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode) */ static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, uint8_t *ecc_code) +{

•   u32 val;
  

•   val = readl(&gpmc_cfg->ecc1_result);
  

•   ecc_code[0] = val & 0xFF;
  

•   ecc_code[1] = (val >> 16) & 0xFF;
  

•   ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
  

•   return 0;
  

+}

+/* GPMC ecc engine settings for read */ +#define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ +#define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ +#define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ +#define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ +#define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */

+/* GPMC ecc engine settings for write */ +#define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ +#define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */ +#define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */

+/**

• • omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
• • @mtd: MTD device structure
• • @mode: Read/Write mode
• • When using BCH with SW correction (i.e. no ELM), sector size is set
• • to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode
• • for both reading and writing with:
• • eccsize0 = 0 (no additional protected byte in spare area)
• • eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area)
• */

+static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd,

•                                            int mode)
  

+{

•   unsigned int bch_type;
  

•   unsigned int dev_width, nsectors;
  

•   struct nand_chip *chip = mtd_to_nand(mtd);
  

•   struct omap_nand_info *info = nand_get_controller_data(chip);
  

•   u32 val, wr_mode;
  

•   unsigned int ecc_size1, ecc_size0;
  

•   /* GPMC configurations for calculating ECC */
  

•   switch (info->ecc_scheme) {
  

•   case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
  

•           bch_type = 1;
  

•           nsectors = 1;
  

•           wr_mode   = BCH_WRAPMODE_6;
  

•           ecc_size0 = BCH_ECC_SIZE0;
  

•           ecc_size1 = BCH_ECC_SIZE1;
  

•           break;
  

•   case OMAP_ECC_BCH8_CODE_HW:
  

•           bch_type = 1;
  

•           nsectors = chip->ecc.steps;
  

•           if (mode == NAND_ECC_READ) {
  

•                   wr_mode   = BCH_WRAPMODE_1;
  

•                   ecc_size0 = BCH8R_ECC_SIZE0;
  

•                   ecc_size1 = BCH8R_ECC_SIZE1;
  

•           } else {
  

•                   wr_mode   = BCH_WRAPMODE_6;
  

•                   ecc_size0 = BCH_ECC_SIZE0;
  

•                   ecc_size1 = BCH_ECC_SIZE1;
  

•           }
  

•           break;
  

•   case OMAP_ECC_BCH16_CODE_HW:
  

•           bch_type = 0x2;
  

•           nsectors = chip->ecc.steps;
  

•           if (mode == NAND_ECC_READ) {
  

•                   wr_mode   = 0x01;
  

•                   ecc_size0 = 52; /* ECC bits in nibbles per sector */
  

•                   ecc_size1 = 0;  /* non-ECC bits in nibbles per sector */
  

•           } else {
  

•                   wr_mode   = 0x01;
  

•                   ecc_size0 = 0;  /* extra bits in nibbles per sector */
  

•                   ecc_size1 = 52; /* OOB bits in nibbles per sector */
  

•           }
  

•           break;
  

•   default:
  

•           return;
  

•   }
  

•   writel(ECCRESULTREG1, &gpmc_cfg->ecc_control);
  

•   /* Configure ecc size for BCH */
  

•   val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT);
  

•   writel(val, &gpmc_cfg->ecc_size_config);
  

•   dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
  

•   /* BCH configuration */
  

•   val = ((1                       << 16) | /* enable BCH */
  

•          (bch_type                << 12) | /* BCH4/BCH8/BCH16 */
  

•          (wr_mode                 <<  8) | /* wrap mode */
  

•          (dev_width               <<  7) | /* bus width */
  

•          (((nsectors - 1) & 0x7)  <<  4) | /* number of sectors */
  

•          (info->cs                <<  1) | /* ECC CS */
  

•          (0x1));                           /* enable ECC */
  

•   writel(val, &gpmc_cfg->ecc_config);
  

•   /* Clear ecc and enable bits */
  

•   writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control);
  

+}

+/**

• • _omap_calculate_ecc_bch - Generate BCH ECC bytes for one sector
• • @mtd: MTD device structure
• • @dat: The pointer to data on which ecc is computed
• • @ecc_code: The ecc_code buffer
• • @sector: The sector number (for a multi sector page)
• • Support calculating of BCH4/8/16 ECC vectors for one sector
• • within a page. Sector number is in @sector.
• */

+static int _omap_calculate_ecc_bch(struct mtd_info *mtd, const u8 *dat,

•                              u8 *ecc_code, int sector)
  

{ struct nand_chip *chip = mtd_to_nand(mtd); struct omap_nand_info *info = nand_get_controller_data(chip); @@ -278,17 +362,11 @@ static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, int8_t i = 0, j;

    switch (info->ecc_scheme) {

•   case OMAP_ECC_HAM1_CODE_HW:
  

•           val = readl(&gpmc_cfg->ecc1_result);
  

•           ecc_code[0] = val & 0xFF;
  

•           ecc_code[1] = (val >> 16) & 0xFF;
  

•           ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0);
  

•           break;
  

#ifdef CONFIG_BCH case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: #endif case OMAP_ECC_BCH8_CODE_HW:

•           ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3];
  

•           ptr = &gpmc_cfg->bch_result_0_3[sector].bch_result_x[3];
            val = readl(ptr);
            ecc_code[i++] = (val >>  0) & 0xFF;
            ptr--;
  

@@ -300,23 +378,24 @@ static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, ecc_code[i++] = (val >> 0) & 0xFF; ptr--; }

•           break;
    case OMAP_ECC_BCH16_CODE_HW:
  

•           val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]);
  

•           val = readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[2]);
            ecc_code[i++] = (val >>  8) & 0xFF;
            ecc_code[i++] = (val >>  0) & 0xFF;
  

•           val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]);
  

•           val = readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[1]);
            ecc_code[i++] = (val >> 24) & 0xFF;
            ecc_code[i++] = (val >> 16) & 0xFF;
            ecc_code[i++] = (val >>  8) & 0xFF;
            ecc_code[i++] = (val >>  0) & 0xFF;
  

•           val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]);
  

•           val = readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[0]);
            ecc_code[i++] = (val >> 24) & 0xFF;
            ecc_code[i++] = (val >> 16) & 0xFF;
            ecc_code[i++] = (val >>  8) & 0xFF;
            ecc_code[i++] = (val >>  0) & 0xFF;
            for (j = 3; j >= 0; j--) {
  

•                   val = readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j]
  

•                   val = readl(&gpmc_cfg->bch_result_0_3[sector].bch_result_x[j]
                                                                    );
                    ecc_code[i++] = (val >> 24) & 0xFF;
                    ecc_code[i++] = (val >> 16) & 0xFF;
  

@@ -329,18 +408,18 @@ static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, } /* ECC scheme specific syndrome customizations */ switch (info->ecc_scheme) {

•   case OMAP_ECC_HAM1_CODE_HW:
  

•           break;
  

#ifdef CONFIG_BCH case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:

•           /* Add constant polynomial to remainder, so that
  

•            * ECC of blank pages results in 0x0 on reading back
  

•            */
            for (i = 0; i < chip->ecc.bytes; i++)
  

•                   *(ecc_code + i) = *(ecc_code + i) ^
  

•                                           bch8_polynomial[i];
  

•                   ecc_code[i] ^= bch8_polynomial[i];
            break;
  

#endif case OMAP_ECC_BCH8_CODE_HW:

•           ecc_code[chip->ecc.bytes - 1] = 0x00;
  

•           /* Set 14th ECC byte as 0x0 for ROM compatibility */
  

•           ecc_code[chip->ecc.bytes - 1] = 0x0;
            break;
    case OMAP_ECC_BCH16_CODE_HW:
            break;
  

@@ -350,6 +429,22 @@ static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, return 0; }

+/**

• • omap_calculate_ecc_bch - ECC generator for 1 sector
• • @mtd: MTD device structure
• • @dat: The pointer to data on which ecc is computed
• • @ecc_code: The ecc_code buffer
• • Support calculating of BCH4/8/16 ECC vectors for one sector. This is used
• • when SW based correction is required as ECC is required for one sector
• • at a time.
• */

+static int omap_calculate_ecc_bch(struct mtd_info *mtd,

•                             const u_char *dat, u_char *ecc_calc)
  

-static int omap_calculate_ecc_bch(struct mtd_info *mtd, +static int __maybe_unused omap_calculate_ecc_bch(struct mtd_info *mtd, const u_char *dat, u_char *ecc_calc)

Acked-by: Michael Trimarchi michael@amarulasolutions.com

+{

•   return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0);
  

+}

static inline void omap_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len) { struct nand_chip *chip = mtd_to_nand(mtd); @@ -474,6 +569,35 @@ static void omap_nand_read_prefetch(struct mtd_info *mtd, uint8_t *buf, int len) #endif /* CONFIG_NAND_OMAP_GPMC_PREFETCH */

#ifdef CONFIG_NAND_OMAP_ELM

+/**

• • omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors
• • @mtd: MTD device structure
• • @dat: The pointer to data on which ecc is computed
• • @ecc_code: The ecc_code buffer
• • Support calculating of BCH4/8/16 ecc vectors for the entire page in one go.
• */

+static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd,

•                                   const u_char *dat, u_char *ecc_calc)
  

+{

•   struct nand_chip *chip = mtd_to_nand(mtd);
  

•   int eccbytes = chip->ecc.bytes;
  

•   unsigned long nsectors;
  

•   int i, ret;
  

•   nsectors = ((readl(&gpmc_cfg->ecc_config) >> 4) & 0x7) + 1;
  

•   for (i = 0; i < nsectors; i++) {
  

•           ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i);
  

•           if (ret)
  

•                   return ret;
  

•           ecc_calc += eccbytes;
  

•   }
  

•   return 0;
  

+}

/*

• omap_reverse_list - re-orders list elements in reverse order [internal]
• @list: pointer to start of list

@@ -626,52 +750,49 @@ static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip, { int i, eccsize = chip->ecc.size; int eccbytes = chip->ecc.bytes;

•   int ecctotal = chip->ecc.total;
    int eccsteps = chip->ecc.steps;
    uint8_t *p = buf;
    uint8_t *ecc_calc = chip->buffers->ecccalc;
    uint8_t *ecc_code = chip->buffers->ecccode;
    uint32_t *eccpos = chip->ecc.layout->eccpos;
    uint8_t *oob = chip->oob_poi;
  

•   uint32_t data_pos;
    uint32_t oob_pos;
  
  

•   data_pos = 0;
    /* oob area start */
    oob_pos = (eccsize * eccsteps) + chip->ecc.layout->eccpos[0];
    oob += chip->ecc.layout->eccpos[0];
  
  

•   for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize,
  

•                           oob += eccbytes) {
  

•           chip->ecc.hwctl(mtd, NAND_ECC_READ);
  

•           /* read data */
  

•           chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1);
  

•           chip->read_buf(mtd, p, eccsize);
  

•           /* read respective ecc from oob area */
  

•           chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
  

•           chip->read_buf(mtd, oob, eccbytes);
  

•           /* read syndrome */
  

•           chip->ecc.calculate(mtd, p, &ecc_calc[i]);
  

•           data_pos += eccsize;
  

•           oob_pos += eccbytes;
  

•   }
  

•   /* Enable ECC engine */
  

•   chip->ecc.hwctl(mtd, NAND_ECC_READ);
  

•   /* read entire page */
  

•   chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
  

•   chip->read_buf(mtd, buf, mtd->writesize);
  

•   /* read all ecc bytes from oob area */
  

•   chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1);
  

•   chip->read_buf(mtd, oob, ecctotal);
  

•   /* Calculate ecc bytes */
  

•   omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc);
  
    for (i = 0; i < chip->ecc.total; i++)
            ecc_code[i] = chip->oob_poi[eccpos[i]];
  
  

•   /* error detect & correct */
    eccsteps = chip->ecc.steps;
    p = buf;
  
    for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
            int stat;
  

•           stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]);
            if (stat < 0)
                    mtd->ecc_stats.failed++;
            else
                    mtd->ecc_stats.corrected += stat;
    }
  

•   return 0;
  

} #endif /* CONFIG_NAND_OMAP_ELM */ @@ -819,9 +940,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, nand->ecc.strength = 8; nand->ecc.size = SECTOR_BYTES; nand->ecc.bytes = 13;

•           nand->ecc.hwctl         = omap_enable_hwecc;
  

•           nand->ecc.hwctl         = omap_enable_hwecc_bch;
            nand->ecc.correct       = omap_correct_data_bch_sw;
  

•           nand->ecc.calculate     = omap_calculate_ecc;
  

•           nand->ecc.calculate     = omap_calculate_ecc_bch;
            /* define ecc-layout */
            ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
            ecclayout->eccpos[0]    = BADBLOCK_MARKER_LENGTH;
  

@@ -860,9 +981,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, nand->ecc.strength = 8; nand->ecc.size = SECTOR_BYTES; nand->ecc.bytes = 14;

•           nand->ecc.hwctl         = omap_enable_hwecc;
  

•           nand->ecc.hwctl         = omap_enable_hwecc_bch;
            nand->ecc.correct       = omap_correct_data_bch;
  

•           nand->ecc.calculate     = omap_calculate_ecc;
  

•           nand->ecc.calculate     = omap_calculate_ecc_bch;
            nand->ecc.read_page     = omap_read_page_bch;
            /* define ecc-layout */
            ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
  

@@ -893,9 +1014,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, nand->ecc.size = SECTOR_BYTES; nand->ecc.bytes = 26; nand->ecc.strength = 16;

•           nand->ecc.hwctl         = omap_enable_hwecc;
  

•           nand->ecc.hwctl         = omap_enable_hwecc_bch;
            nand->ecc.correct       = omap_correct_data_bch;
  

•           nand->ecc.calculate     = omap_calculate_ecc;
  

•           nand->ecc.calculate     = omap_calculate_ecc_bch;
            nand->ecc.read_page     = omap_read_page_bch;
            /* define ecc-layout */
            ecclayout->eccbytes     = nand->ecc.bytes * eccsteps;
  

-- 2.34.1