18 #include <linux/mtd/nand-ecc-sw-bch.h>
122 /* GPMC ecc engine settings for read */
129 /* GPMC ecc engine settings for write */
170 	/* fields specific for BCHx_HW ECC scheme */
723  * gen_true_ecc - This function will generate true ECC value
724  * @ecc_buf: buffer to store ecc code
726  * This generated true ECC value can be used when correcting
744  * @ecc_data1:  ecc code from nand spare area
745  * @ecc_data2:  ecc code from hardware register obtained from hardware ecc
748  * This function compares two ECC's and indicates if there is an error.
824 		 *  ECC values are equal  in omap_compare_ecc()
830 		pr_debug("ECC UNCORRECTED_ERROR 1\n");  in omap_compare_ecc()
835 		pr_debug("ECC UNCORRECTED_ERROR B\n");  in omap_compare_ecc()
852 		pr_debug("Correcting single bit ECC error at offset: "  in omap_compare_ecc()
871  * omap_correct_data - Compares the ECC read with HW generated ECC
874  * @read_ecc: ecc read from nand flash
875  * @calc_ecc: ecc read from HW ECC registers
877  * Compares the ecc read from nand spare area with ECC registers values
878  * and if ECC's mismatched, it will call 'omap_compare_ecc' for error
892 	if (info->nand.ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST &&  in omap_correct_data()
893 	    info->nand.ecc.size == 2048)  in omap_correct_data()
914  * omap_calculate_ecc - Generate non-inverted ECC bytes.
916  * @dat: The pointer to data on which ecc is computed
919  * Using noninverted ECC can be considered ugly since writing a blank
920  * page ie. padding will clear the ECC bytes. This is no problem as long
922  * an erased page will produce an ECC mismatch between generated and read
923  * ECC bytes that has to be dealt with separately.
935 	/* read ecc result */  in omap_calculate_ecc()
946  * omap_enable_hwecc - This function enables the hardware ecc functionality
956 	/* clear ecc and enable bits */  in omap_enable_hwecc()
960 	/* program ecc and result sizes */  in omap_enable_hwecc()
961 	val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |  in omap_enable_hwecc()
979 	/* (ECC 16 or 8 bit col) | ( CS  )  | ECC Enable */  in omap_enable_hwecc()
1029  * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
1049 	/* GPMC configurations for calculating ECC */  in omap_enable_hwecc_bch()
1060 		nsectors = chip->ecc.steps;  in omap_enable_hwecc_bch()
1080 		nsectors = chip->ecc.steps;  in omap_enable_hwecc_bch()
1093 		nsectors = chip->ecc.steps;  in omap_enable_hwecc_bch()
1096 			ecc_size0 = 52; /* ECC bits in nibbles per sector */  in omap_enable_hwecc_bch()
1097 			ecc_size1 = 0;  /* non-ECC bits in nibbles per sector */  in omap_enable_hwecc_bch()
1110 	/* Configure ecc size for BCH */  in omap_enable_hwecc_bch()
1122 	       (info->gpmc_cs            <<  1) | /* ECC CS */  in omap_enable_hwecc_bch()
1123 	       (0x1));                            /* enable ECC */  in omap_enable_hwecc_bch()
1127 	/* Clear ecc and enable bits */  in omap_enable_hwecc_bch()
1136  * _omap_calculate_ecc_bch - Generate ECC bytes for one sector
1138  * @dat:	The pointer to data on which ecc is computed
1142  * Support calculating of BCH4/8/16 ECC vectors for one sector
1149 	int eccbytes	= info->nand.ecc.bytes;  in _omap_calculate_ecc_bch()
1230 	/* ECC scheme specific syndrome customizations */  in _omap_calculate_ecc_bch()
1234 		 * ECC of blank pages results in 0x0 on reading back  in _omap_calculate_ecc_bch()
1240 		/* Set  8th ECC byte as 0x0 for ROM compatibility */  in _omap_calculate_ecc_bch()
1245 		 * ECC of blank pages results in 0x0 on reading back  in _omap_calculate_ecc_bch()
1251 		/* Set 14th ECC byte as 0x0 for ROM compatibility */  in _omap_calculate_ecc_bch()
1264  * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction
1266  * @dat:	The pointer to data on which ecc is computed
1267  * @ecc_calc:	Buffer storing the calculated ECC bytes
1269  * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used
1270  * when SW based correction is required as ECC is required for one sector
1280  * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors
1282  * @dat:	The pointer to data on which ecc is computed
1283  * @ecc_calc:	Buffer storing the calculated ECC bytes
1285  * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go.
1291 	int eccbytes = info->nand.ecc.bytes;  in omap_calculate_ecc_bch_multi()
1322 	for (i = 0; i < info->nand.ecc.size; i++) {  in erased_sector_bitflips()
1324 		if (flip_bits > info->nand.ecc.strength)  in erased_sector_bitflips()
1328 	for (i = 0; i < info->nand.ecc.bytes - 1; i++) {  in erased_sector_bitflips()
1330 		if (flip_bits > info->nand.ecc.strength)  in erased_sector_bitflips()
1339 		memset(data, 0xFF, info->nand.ecc.size);  in erased_sector_bitflips()
1340 		memset(oob, 0xFF, info->nand.ecc.bytes);  in erased_sector_bitflips()
1350  * @read_ecc:	ecc read from nand flash
1351  * @calc_ecc:	ecc read from HW ECC registers
1353  * Calculated ecc vector reported as zero in case of non-error pages.
1354  * In case of non-zero ecc vector, first filter out erased-pages, and
1361 	struct nand_ecc_ctrl *ecc = &info->nand.ecc;  in omap_elm_correct_data()  local
1377 		/* omit  7th ECC byte reserved for ROM code compatibility */  in omap_elm_correct_data()
1378 		actual_eccbytes = ecc->bytes - 1;  in omap_elm_correct_data()
1382 		/* omit 14th ECC byte reserved for ROM code compatibility */  in omap_elm_correct_data()
1383 		actual_eccbytes = ecc->bytes - 1;  in omap_elm_correct_data()
1387 		actual_eccbytes = ecc->bytes;  in omap_elm_correct_data()
1403 		 * In case of error, non zero ecc reported.  in omap_elm_correct_data()
1407 				eccflag = 1; /* non zero ecc, error present */  in omap_elm_correct_data()
1416 				 * calc_ecc[] matches pattern for ECC(all 0xff)  in omap_elm_correct_data()
1420 				buf = &data[info->nand.ecc.size * i];  in omap_elm_correct_data()
1430 					 * number of 0-bits within ECC limits  in omap_elm_correct_data()
1447 		/* Update the ecc vector */  in omap_elm_correct_data()
1448 		calc_ecc += ecc->bytes;  in omap_elm_correct_data()
1449 		read_ecc += ecc->bytes;  in omap_elm_correct_data()
1480 				error_max = (ecc->size + actual_eccbytes) * 8;  in omap_elm_correct_data()
1512 		data += ecc->size;  in omap_elm_correct_data()
1513 		spare_ecc += ecc->bytes;  in omap_elm_correct_data()
1520  * omap_write_page_bch - BCH ecc based write page function for entire page
1533 	uint8_t *ecc_calc = chip->ecc.calc_buf;  in omap_write_page_bch()
1542 		/* Enable GPMC ecc engine */  in omap_write_page_bch()
1543 		chip->ecc.hwctl(chip, NAND_ECC_WRITE);  in omap_write_page_bch()
1549 		/* Update ecc vector from GPMC result registers */  in omap_write_page_bch()
1564 	/* Write ecc vector to OOB area */  in omap_write_page_bch()
1571  * omap_write_subpage_bch - BCH hardware ECC based subpage write
1587 	u8 *ecc_calc = chip->ecc.calc_buf;  in omap_write_subpage_bch()
1588 	int ecc_size      = chip->ecc.size;  in omap_write_subpage_bch()
1589 	int ecc_bytes     = chip->ecc.bytes;  in omap_write_subpage_bch()
1597 	 * as ECC is calculated by hardware.  in omap_write_subpage_bch()
1598 	 * ECC is calculated for all subpages but we choose  in omap_write_subpage_bch()
1606 		/* Enable GPMC ECC engine */  in omap_write_subpage_bch()
1607 		chip->ecc.hwctl(chip, NAND_ECC_WRITE);  in omap_write_subpage_bch()
1617 			/* Mask ECC of un-touched subpages with 0xFFs */  in omap_write_subpage_bch()
1633 		 * Copy the calculated ECC for the whole page including the  in omap_write_subpage_bch()
1650  * omap_read_page_bch - BCH ecc based page read function for entire page
1656  * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module
1660  * ecc engine enabled. ecc vector updated after read of OOB data.
1661  * For non error pages ecc vector reported as zero.
1668 	uint8_t *ecc_calc = chip->ecc.calc_buf;  in omap_read_page_bch()
1669 	uint8_t *ecc_code = chip->ecc.code_buf;  in omap_read_page_bch()
1678 		/* Enable GPMC ecc engine */  in omap_read_page_bch()
1679 		chip->ecc.hwctl(chip, NAND_ECC_READ);  in omap_read_page_bch()
1698 		/* Calculate ecc bytes */  in omap_read_page_bch()
1712 		stat = chip->ecc.correct(chip,  in omap_read_page_bch()
1824 	/* select ecc-scheme for NAND */  in omap_get_dt_info()
1825 	if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) {  in omap_get_dt_info()
1826 		dev_err(dev, "ti,nand-ecc-opt not found\n");  in omap_get_dt_info()
1848 		dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n");  in omap_get_dt_info()
1883 	oobregion->length = chip->ecc.total;  in omap_ooblayout_ecc()
1902 	off += chip->ecc.total;  in omap_ooblayout_free()
1913 	.ecc = omap_ooblayout_ecc,
1930 	 * reserved after each ECC step.  in omap_sw_ooblayout_ecc()
1951 	 * reserved after each ECC step.  in omap_sw_ooblayout_free()
1964 	.ecc = omap_sw_ooblayout_ecc,
2070 		chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT;  in omap_nand_attach_chip()
2071 		chip->ecc.algo = NAND_ECC_ALGO_HAMMING;  in omap_nand_attach_chip()
2075 	/* Populate MTD interface based on ECC scheme */  in omap_nand_attach_chip()
2079 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2080 		chip->ecc.bytes		= 3;  in omap_nand_attach_chip()
2081 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2082 		chip->ecc.strength	= 1;  in omap_nand_attach_chip()
2083 		chip->ecc.calculate	= omap_calculate_ecc;  in omap_nand_attach_chip()
2084 		chip->ecc.hwctl		= omap_enable_hwecc;  in omap_nand_attach_chip()
2085 		chip->ecc.correct	= omap_correct_data;  in omap_nand_attach_chip()
2087 		oobbytes_per_step	= chip->ecc.bytes;  in omap_nand_attach_chip()
2096 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2097 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2098 		chip->ecc.bytes		= 7;  in omap_nand_attach_chip()
2099 		chip->ecc.strength	= 4;  in omap_nand_attach_chip()
2100 		chip->ecc.hwctl		= omap_enable_hwecc_bch;  in omap_nand_attach_chip()
2101 		chip->ecc.correct	= rawnand_sw_bch_correct;  in omap_nand_attach_chip()
2102 		chip->ecc.calculate	= omap_calculate_ecc_bch_sw;  in omap_nand_attach_chip()
2105 		oobbytes_per_step	= chip->ecc.bytes + 1;  in omap_nand_attach_chip()
2115 		pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n");  in omap_nand_attach_chip()
2116 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2117 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2119 		chip->ecc.bytes		= 7 + 1;  in omap_nand_attach_chip()
2120 		chip->ecc.strength	= 4;  in omap_nand_attach_chip()
2121 		chip->ecc.hwctl		= omap_enable_hwecc_bch;  in omap_nand_attach_chip()
2122 		chip->ecc.correct	= omap_elm_correct_data;  in omap_nand_attach_chip()
2123 		chip->ecc.read_page	= omap_read_page_bch;  in omap_nand_attach_chip()
2124 		chip->ecc.write_page	= omap_write_page_bch;  in omap_nand_attach_chip()
2125 		chip->ecc.write_subpage	= omap_write_subpage_bch;  in omap_nand_attach_chip()
2127 		oobbytes_per_step	= chip->ecc.bytes;  in omap_nand_attach_chip()
2133 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2134 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2135 		chip->ecc.bytes		= 13;  in omap_nand_attach_chip()
2136 		chip->ecc.strength	= 8;  in omap_nand_attach_chip()
2137 		chip->ecc.hwctl		= omap_enable_hwecc_bch;  in omap_nand_attach_chip()
2138 		chip->ecc.correct	= rawnand_sw_bch_correct;  in omap_nand_attach_chip()
2139 		chip->ecc.calculate	= omap_calculate_ecc_bch_sw;  in omap_nand_attach_chip()
2142 		oobbytes_per_step	= chip->ecc.bytes + 1;  in omap_nand_attach_chip()
2152 		pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n");  in omap_nand_attach_chip()
2153 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2154 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2156 		chip->ecc.bytes		= 13 + 1;  in omap_nand_attach_chip()
2157 		chip->ecc.strength	= 8;  in omap_nand_attach_chip()
2158 		chip->ecc.hwctl		= omap_enable_hwecc_bch;  in omap_nand_attach_chip()
2159 		chip->ecc.correct	= omap_elm_correct_data;  in omap_nand_attach_chip()
2160 		chip->ecc.read_page	= omap_read_page_bch;  in omap_nand_attach_chip()
2161 		chip->ecc.write_page	= omap_write_page_bch;  in omap_nand_attach_chip()
2162 		chip->ecc.write_subpage	= omap_write_subpage_bch;  in omap_nand_attach_chip()
2164 		oobbytes_per_step	= chip->ecc.bytes;  in omap_nand_attach_chip()
2169 		pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n");  in omap_nand_attach_chip()
2170 		chip->ecc.engine_type	= NAND_ECC_ENGINE_TYPE_ON_HOST;  in omap_nand_attach_chip()
2171 		chip->ecc.size		= 512;  in omap_nand_attach_chip()
2172 		chip->ecc.bytes		= 26;  in omap_nand_attach_chip()
2173 		chip->ecc.strength	= 16;  in omap_nand_attach_chip()
2174 		chip->ecc.hwctl		= omap_enable_hwecc_bch;  in omap_nand_attach_chip()
2175 		chip->ecc.correct	= omap_elm_correct_data;  in omap_nand_attach_chip()
2176 		chip->ecc.read_page	= omap_read_page_bch;  in omap_nand_attach_chip()
2177 		chip->ecc.write_page	= omap_write_page_bch;  in omap_nand_attach_chip()
2178 		chip->ecc.write_subpage	= omap_write_subpage_bch;  in omap_nand_attach_chip()
2180 		oobbytes_per_step	= chip->ecc.bytes;  in omap_nand_attach_chip()
2184 		dev_err(dev, "Invalid or unsupported ECC scheme\n");  in omap_nand_attach_chip()
2189 		chip->ecc.steps = mtd->writesize / chip->ecc.size;  in omap_nand_attach_chip()
2190 		info->neccpg = chip->ecc.steps / ERROR_VECTOR_MAX;  in omap_nand_attach_chip()
2195 			info->nsteps_per_eccpg = chip->ecc.steps;  in omap_nand_attach_chip()
2197 		info->eccpg_size = info->nsteps_per_eccpg * chip->ecc.size;  in omap_nand_attach_chip()
2198 		info->eccpg_bytes = info->nsteps_per_eccpg * chip->ecc.bytes;  in omap_nand_attach_chip()
2201 				 info->nsteps_per_eccpg, chip->ecc.size,  in omap_nand_attach_chip()
2202 				 chip->ecc.bytes);  in omap_nand_attach_chip()
2207 	/* Check if NAND device's OOB is enough to store ECC signatures */  in omap_nand_attach_chip()
2209 			 (mtd->writesize / chip->ecc.size));  in omap_nand_attach_chip()
2224 /* Shared among all NAND instances to synchronize access to the ECC Engine */