1 // SPDX-License-Identifier: GPL-2.0+
3  * Generic Error-Correcting Code (ECC) engine
10  * This file describes the abstraction of any NAND ECC engine. It has been
11  * designed to fit most cases, including parallel NANDs and SPI-NANDs.
13  * There are three main situations where instantiating this ECC engine makes
15  *   - external: The ECC engine is outside the NAND pipeline, typically this
16  *               is a software ECC engine, or an hardware engine that is
17  *               outside the NAND controller pipeline.
18  *   - pipelined: The ECC engine is inside the NAND pipeline, ie. on the
19  *                controller's side. This is the case of most of the raw NAND
20  *                controllers. In the pipeline case, the ECC bytes are
23  *   - ondie: The ECC engine is inside the NAND pipeline, on the chip's side.
24  *            Some NAND chips can correct themselves the data.
28  *   - prepare: Prepare an I/O request. Enable/disable the ECC engine based on
30  *              engine, this step may involve to derive the ECC bytes and place
32  *   - finish: Finish an I/O request. Correct the data in case of a read
35  *             hardware specific stuff to do, like shutting down the engine to
40  *   - raw: Correction disabled
41  *   - ecc: Correction enabled
44  *   - read: Load data from the NAND chip
45  *   - write: Store data in the NAND chip
51  * [external ECC engine]
52  *   - external + prepare + raw + read: do nothing
53  *   - external + finish  + raw + read: do nothing
54  *   - external + prepare + raw + write: do nothing
55  *   - external + finish  + raw + write: do nothing
56  *   - external + prepare + ecc + read: do nothing
57  *   - external + finish  + ecc + read: calculate expected ECC bytes, extract
58  *                                      ECC bytes from OOB buffer, correct
60  *   - external + prepare + ecc + write: calculate ECC bytes and store them at
63  *   - external + finish  + ecc + write: do nothing
65  * [pipelined ECC engine]
66  *   - pipelined + prepare + raw + read: disable the controller's ECC engine if
68  *   - pipelined + finish  + raw + read: do nothing
69  *   - pipelined + prepare + raw + write: disable the controller's ECC engine if
71  *   - pipelined + finish  + raw + write: do nothing
72  *   - pipelined + prepare + ecc + read: enable the controller's ECC engine if
74  *   - pipelined + finish  + ecc + read: check the status, report any
76  *   - pipelined + prepare + ecc + write: enable the controller's ECC engine if
78  *   - pipelined + finish  + ecc + write: do nothing
80  * [ondie ECC engine]
81  *   - ondie + prepare + raw + read: send commands to disable the on-chip ECC
82  *                                   engine if activated
83  *   - ondie + finish  + raw + read: do nothing
84  *   - ondie + prepare + raw + write: send commands to disable the on-chip ECC
85  *                                    engine if activated
86  *   - ondie + finish  + raw + write: do nothing
87  *   - ondie + prepare + ecc + read: send commands to enable the on-chip ECC
88  *                                   engine if deactivated
89  *   - ondie + finish  + ecc + read: send commands to check the status, report
91  *   - ondie + prepare + ecc + write: send commands to enable the on-chip ECC
92  *                                    engine if deactivated
93  *   - ondie + finish  + ecc + write: do nothing
97 #include <linux/mtd/nand.h>
107  * nand_ecc_init_ctx - Init the ECC engine context
108  * @nand: the NAND device
112 int nand_ecc_init_ctx(struct nand_device *nand)  in nand_ecc_init_ctx()  argument
114 	if (!nand->ecc.engine || !nand->ecc.engine->ops->init_ctx)  in nand_ecc_init_ctx()
117 	return nand->ecc.engine->ops->init_ctx(nand);  in nand_ecc_init_ctx()
122  * nand_ecc_cleanup_ctx - Cleanup the ECC engine context
123  * @nand: the NAND device
125 void nand_ecc_cleanup_ctx(struct nand_device *nand)  in nand_ecc_cleanup_ctx()  argument
127 	if (nand->ecc.engine && nand->ecc.engine->ops->cleanup_ctx)  in nand_ecc_cleanup_ctx()
128 		nand->ecc.engine->ops->cleanup_ctx(nand);  in nand_ecc_cleanup_ctx()
133  * nand_ecc_prepare_io_req - Prepare an I/O request
134  * @nand: the NAND device
137 int nand_ecc_prepare_io_req(struct nand_device *nand,  in nand_ecc_prepare_io_req()  argument
140 	if (!nand->ecc.engine || !nand->ecc.engine->ops->prepare_io_req)  in nand_ecc_prepare_io_req()
143 	return nand->ecc.engine->ops->prepare_io_req(nand, req);  in nand_ecc_prepare_io_req()
148  * nand_ecc_finish_io_req - Finish an I/O request
149  * @nand: the NAND device
152 int nand_ecc_finish_io_req(struct nand_device *nand,  in nand_ecc_finish_io_req()  argument
155 	if (!nand->ecc.engine || !nand->ecc.engine->ops->finish_io_req)  in nand_ecc_finish_io_req()
158 	return nand->ecc.engine->ops->finish_io_req(nand, req);  in nand_ecc_finish_io_req()
166 	struct nand_device *nand = mtd_to_nanddev(mtd);  in nand_ooblayout_ecc_sp()  local
167 	unsigned int total_ecc_bytes = nand->ecc.ctx.total;  in nand_ooblayout_ecc_sp()
170 		return -ERANGE;  in nand_ooblayout_ecc_sp()
173 		oobregion->offset = 0;  in nand_ooblayout_ecc_sp()
174 		if (mtd->oobsize == 16)  in nand_ooblayout_ecc_sp()
175 			oobregion->length = 4;  in nand_ooblayout_ecc_sp()
177 			oobregion->length = 3;  in nand_ooblayout_ecc_sp()
179 		if (mtd->oobsize == 8)  in nand_ooblayout_ecc_sp()
180 			return -ERANGE;  in nand_ooblayout_ecc_sp()
182 		oobregion->offset = 6;  in nand_ooblayout_ecc_sp()
183 		oobregion->length = total_ecc_bytes - 4;  in nand_ooblayout_ecc_sp()
193 		return -ERANGE;  in nand_ooblayout_free_sp()
195 	if (mtd->oobsize == 16) {  in nand_ooblayout_free_sp()
197 			return -ERANGE;  in nand_ooblayout_free_sp()
199 		oobregion->length = 8;  in nand_ooblayout_free_sp()
200 		oobregion->offset = 8;  in nand_ooblayout_free_sp()
202 		oobregion->length = 2;  in nand_ooblayout_free_sp()
204 			oobregion->offset = 3;  in nand_ooblayout_free_sp()
206 			oobregion->offset = 6;  in nand_ooblayout_free_sp()
213 	.ecc = nand_ooblayout_ecc_sp,
226 	struct nand_device *nand = mtd_to_nanddev(mtd);  in nand_ooblayout_ecc_lp()  local
227 	unsigned int total_ecc_bytes = nand->ecc.ctx.total;  in nand_ooblayout_ecc_lp()
230 		return -ERANGE;  in nand_ooblayout_ecc_lp()
232 	oobregion->length = total_ecc_bytes;  in nand_ooblayout_ecc_lp()
233 	oobregion->offset = mtd->oobsize - oobregion->length;  in nand_ooblayout_ecc_lp()
241 	struct nand_device *nand = mtd_to_nanddev(mtd);  in nand_ooblayout_free_lp()  local
242 	unsigned int total_ecc_bytes = nand->ecc.ctx.total;  in nand_ooblayout_free_lp()
245 		return -ERANGE;  in nand_ooblayout_free_lp()
247 	oobregion->length = mtd->oobsize - total_ecc_bytes - 2;  in nand_ooblayout_free_lp()
248 	oobregion->offset = 2;  in nand_ooblayout_free_lp()
254 	.ecc = nand_ooblayout_ecc_lp,
265  * Support the old "large page" layout used for 1-bit Hamming ECC where ECC
271 	struct nand_device *nand = mtd_to_nanddev(mtd);  in nand_ooblayout_ecc_lp_hamming()  local
272 	unsigned int total_ecc_bytes = nand->ecc.ctx.total;  in nand_ooblayout_ecc_lp_hamming()
275 		return -ERANGE;  in nand_ooblayout_ecc_lp_hamming()
277 	switch (mtd->oobsize) {  in nand_ooblayout_ecc_lp_hamming()
279 		oobregion->offset = 40;  in nand_ooblayout_ecc_lp_hamming()
282 		oobregion->offset = 80;  in nand_ooblayout_ecc_lp_hamming()
285 		return -EINVAL;  in nand_ooblayout_ecc_lp_hamming()
288 	oobregion->length = total_ecc_bytes;  in nand_ooblayout_ecc_lp_hamming()
289 	if (oobregion->offset + oobregion->length > mtd->oobsize)  in nand_ooblayout_ecc_lp_hamming()
290 		return -ERANGE;  in nand_ooblayout_ecc_lp_hamming()
298 	struct nand_device *nand = mtd_to_nanddev(mtd);  in nand_ooblayout_free_lp_hamming()  local
299 	unsigned int total_ecc_bytes = nand->ecc.ctx.total;  in nand_ooblayout_free_lp_hamming()
303 		return -ERANGE;  in nand_ooblayout_free_lp_hamming()
305 	switch (mtd->oobsize) {  in nand_ooblayout_free_lp_hamming()
313 		return -EINVAL;  in nand_ooblayout_free_lp_hamming()
317 		oobregion->offset = 2;  in nand_ooblayout_free_lp_hamming()
318 		oobregion->length = ecc_offset - 2;  in nand_ooblayout_free_lp_hamming()
320 		oobregion->offset = ecc_offset + total_ecc_bytes;  in nand_ooblayout_free_lp_hamming()
321 		oobregion->length = mtd->oobsize - oobregion->offset;  in nand_ooblayout_free_lp_hamming()
328 	.ecc = nand_ooblayout_ecc_lp_hamming,
343 	if (of_property_read_bool(np, "nand-no-ecc-engine"))  in of_get_nand_ecc_engine_type()
346 	if (of_property_read_bool(np, "nand-use-soft-ecc-engine"))  in of_get_nand_ecc_engine_type()
349 	eng_np = of_parse_phandle(np, "nand-ecc-engine", 0);  in of_get_nand_ecc_engine_type()
373 	err = of_property_read_string(np, "nand-ecc-placement", &pm);  in of_get_nand_ecc_placement()
397 	err = of_property_read_string(np, "nand-ecc-algo", &pm);  in of_get_nand_ecc_algo()
415 	ret = of_property_read_u32(np, "nand-ecc-step-size", &val);  in of_get_nand_ecc_step_size()
424 	ret = of_property_read_u32(np, "nand-ecc-strength", &val);  in of_get_nand_ecc_strength()
428 void of_get_nand_ecc_user_config(struct nand_device *nand)  in of_get_nand_ecc_user_config()  argument
430 	struct device_node *dn = nanddev_get_of_node(nand);  in of_get_nand_ecc_user_config()
433 	nand->ecc.user_conf.engine_type = of_get_nand_ecc_engine_type(dn);  in of_get_nand_ecc_user_config()
434 	nand->ecc.user_conf.algo = of_get_nand_ecc_algo(dn);  in of_get_nand_ecc_user_config()
435 	nand->ecc.user_conf.placement = of_get_nand_ecc_placement(dn);  in of_get_nand_ecc_user_config()
439 		nand->ecc.user_conf.strength = strength;  in of_get_nand_ecc_user_config()
443 		nand->ecc.user_conf.step_size = size;  in of_get_nand_ecc_user_config()
445 	if (of_property_read_bool(dn, "nand-ecc-maximize"))  in of_get_nand_ecc_user_config()
446 		nand->ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH;  in of_get_nand_ecc_user_config()
451  * nand_ecc_is_strong_enough - Check if the chip configuration meets the
454  * @nand: Device to check
467 bool nand_ecc_is_strong_enough(struct nand_device *nand)  in nand_ecc_is_strong_enough()  argument
469 	const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand);  in nand_ecc_is_strong_enough()
470 	const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand);  in nand_ecc_is_strong_enough()
471 	struct mtd_info *mtd = nanddev_to_mtd(nand);  in nand_ecc_is_strong_enough()
474 	if (conf->step_size == 0 || reqs->step_size == 0)  in nand_ecc_is_strong_enough()
482 	corr = (mtd->writesize * conf->strength) / conf->step_size;  in nand_ecc_is_strong_enough()
483 	ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size;  in nand_ecc_is_strong_enough()
485 	return corr >= ds_corr && conf->strength >= reqs->strength;  in nand_ecc_is_strong_enough()
489 /* ECC engine driver internal helpers */
491 			       struct nand_device *nand)  in nand_ecc_init_req_tweaking()  argument
495 	ctx->nand = nand;  in nand_ecc_init_req_tweaking()
498 	if (!ctx->page_buffer_size)  in nand_ecc_init_req_tweaking()
499 		ctx->page_buffer_size = nanddev_page_size(nand);  in nand_ecc_init_req_tweaking()
500 	if (!ctx->oob_buffer_size)  in nand_ecc_init_req_tweaking()
501 		ctx->oob_buffer_size = nanddev_per_page_oobsize(nand);  in nand_ecc_init_req_tweaking()
503 	total_buffer_size = ctx->page_buffer_size + ctx->oob_buffer_size;  in nand_ecc_init_req_tweaking()
505 	ctx->spare_databuf = kzalloc(total_buffer_size, GFP_KERNEL);  in nand_ecc_init_req_tweaking()
506 	if (!ctx->spare_databuf)  in nand_ecc_init_req_tweaking()
507 		return -ENOMEM;  in nand_ecc_init_req_tweaking()
509 	ctx->spare_oobbuf = ctx->spare_databuf + ctx->page_buffer_size;  in nand_ecc_init_req_tweaking()
517 	kfree(ctx->spare_databuf);  in nand_ecc_cleanup_req_tweaking()
528 	struct nand_device *nand = ctx->nand;  in nand_ecc_tweak_req()  local
532 	ctx->orig_req = *req;  in nand_ecc_tweak_req()
533 	ctx->bounce_data = false;  in nand_ecc_tweak_req()
534 	ctx->bounce_oob = false;  in nand_ecc_tweak_req()
535 	orig = &ctx->orig_req;  in nand_ecc_tweak_req()
539 	if (orig->datalen < nanddev_page_size(nand)) {  in nand_ecc_tweak_req()
540 		ctx->bounce_data = true;  in nand_ecc_tweak_req()
541 		tweak->dataoffs = 0;  in nand_ecc_tweak_req()
542 		tweak->datalen = nanddev_page_size(nand);  in nand_ecc_tweak_req()
543 		tweak->databuf.in = ctx->spare_databuf;  in nand_ecc_tweak_req()
544 		memset(tweak->databuf.in, 0xFF, ctx->page_buffer_size);  in nand_ecc_tweak_req()
547 	if (orig->ooblen < nanddev_per_page_oobsize(nand)) {  in nand_ecc_tweak_req()
548 		ctx->bounce_oob = true;  in nand_ecc_tweak_req()
549 		tweak->ooboffs = 0;  in nand_ecc_tweak_req()
550 		tweak->ooblen = nanddev_per_page_oobsize(nand);  in nand_ecc_tweak_req()
551 		tweak->oobbuf.in = ctx->spare_oobbuf;  in nand_ecc_tweak_req()
552 		memset(tweak->oobbuf.in, 0xFF, ctx->oob_buffer_size);  in nand_ecc_tweak_req()
556 	if (orig->type == NAND_PAGE_WRITE) {  in nand_ecc_tweak_req()
557 		if (ctx->bounce_data)  in nand_ecc_tweak_req()
558 			memcpy((void *)tweak->databuf.out + orig->dataoffs,  in nand_ecc_tweak_req()
559 			       orig->databuf.out, orig->datalen);  in nand_ecc_tweak_req()
561 		if (ctx->bounce_oob)  in nand_ecc_tweak_req()
562 			memcpy((void *)tweak->oobbuf.out + orig->ooboffs,  in nand_ecc_tweak_req()
563 			       orig->oobbuf.out, orig->ooblen);  in nand_ecc_tweak_req()
573 	orig = &ctx->orig_req;  in nand_ecc_restore_req()
577 	if (orig->type == NAND_PAGE_READ) {  in nand_ecc_restore_req()
578 		if (ctx->bounce_data)  in nand_ecc_restore_req()
579 			memcpy(orig->databuf.in,  in nand_ecc_restore_req()
580 			       tweak->databuf.in + orig->dataoffs,  in nand_ecc_restore_req()
581 			       orig->datalen);  in nand_ecc_restore_req()
583 		if (ctx->bounce_oob)  in nand_ecc_restore_req()
584 			memcpy(orig->oobbuf.in,  in nand_ecc_restore_req()
585 			       tweak->oobbuf.in + orig->ooboffs,  in nand_ecc_restore_req()
586 			       orig->ooblen);  in nand_ecc_restore_req()
594 struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand)  in nand_ecc_get_sw_engine()  argument
596 	unsigned int algo = nand->ecc.user_conf.algo;  in nand_ecc_get_sw_engine()
599 		algo = nand->ecc.defaults.algo;  in nand_ecc_get_sw_engine()
614 struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand)  in nand_ecc_get_on_die_hw_engine()  argument
616 	return nand->ecc.ondie_engine;  in nand_ecc_get_on_die_hw_engine()
620 int nand_ecc_register_on_host_hw_engine(struct nand_ecc_engine *engine)  in nand_ecc_register_on_host_hw_engine()  argument
624 	if (!engine)  in nand_ecc_register_on_host_hw_engine()
625 		return -EINVAL;  in nand_ecc_register_on_host_hw_engine()
627 	/* Prevent multiple registrations of one engine */  in nand_ecc_register_on_host_hw_engine()
629 		if (item == engine)  in nand_ecc_register_on_host_hw_engine()
633 	list_add_tail(&engine->node, &on_host_hw_engines);  in nand_ecc_register_on_host_hw_engine()
640 int nand_ecc_unregister_on_host_hw_engine(struct nand_ecc_engine *engine)  in nand_ecc_unregister_on_host_hw_engine()  argument
642 	if (!engine)  in nand_ecc_unregister_on_host_hw_engine()
643 		return -EINVAL;  in nand_ecc_unregister_on_host_hw_engine()
646 	list_del(&engine->node);  in nand_ecc_unregister_on_host_hw_engine()
658 		if (item->dev == dev)  in nand_ecc_match_on_host_hw_engine()
664 struct nand_ecc_engine *nand_ecc_get_on_host_hw_engine(struct nand_device *nand)  in nand_ecc_get_on_host_hw_engine()  argument
666 	struct nand_ecc_engine *engine = NULL;  in nand_ecc_get_on_host_hw_engine()  local
667 	struct device *dev = &nand->mtd.dev;  in nand_ecc_get_on_host_hw_engine()
674 	/* Check for an explicit nand-ecc-engine property */  in nand_ecc_get_on_host_hw_engine()
675 	np = of_parse_phandle(dev->of_node, "nand-ecc-engine", 0);  in nand_ecc_get_on_host_hw_engine()
679 			return ERR_PTR(-EPROBE_DEFER);  in nand_ecc_get_on_host_hw_engine()
681 		engine = nand_ecc_match_on_host_hw_engine(&pdev->dev);  in nand_ecc_get_on_host_hw_engine()
685 		if (!engine)  in nand_ecc_get_on_host_hw_engine()
686 			return ERR_PTR(-EPROBE_DEFER);  in nand_ecc_get_on_host_hw_engine()
689 	if (engine)  in nand_ecc_get_on_host_hw_engine()
690 		get_device(engine->dev);  in nand_ecc_get_on_host_hw_engine()
692 	return engine;  in nand_ecc_get_on_host_hw_engine()
696 void nand_ecc_put_on_host_hw_engine(struct nand_device *nand)  in nand_ecc_put_on_host_hw_engine()  argument
698 	put_device(nand->ecc.engine->dev);  in nand_ecc_put_on_host_hw_engine()
703  * In the case of a pipelined engine, the device registering the ECC
704  * engine is not necessarily the ECC engine itself but may be a host controller.
706  * which actually represents the ECC engine.
715 	 * it in order to get the right ECC engine device we are looking for.  in nand_ecc_get_engine_dev()
717 	np = of_parse_phandle(host->of_node, "nand-ecc-engine", 0);  in nand_ecc_get_engine_dev()
730 	return &ecc_pdev->dev;  in nand_ecc_get_engine_dev()
735 MODULE_DESCRIPTION("Generic ECC engine");