Lines Matching +full:nand +full:- +full:ecc +full:- +full:strength
1 // SPDX-License-Identifier: GPL-2.0
3 // Driver for the SPI-NAND mode of Mediatek NAND Flash Interface
7 // This driver is based on the SPI-NAND mtd driver from Mediatek SDK:
13 // like the following: (sizeof(FDM + ECC) = snf->nfi_cfg.spare_size)
14 // +---------+------+------+---------+------+------+-----+
16 // +---------+------+------+---------+------+------+-----+
17 // With auto-format turned on, DMA only returns this part:
18 // +---------+---------+-----+
20 // +---------+---------+-----+
21 // The FDM data will be filled to the registers, and ECC parity data isn't
23 // With auto-format off, all ((Sector+FDM+ECC)*nsectors) will be read over DMA
24 // in it's original order shown in the first table. ECC can't be turned on when
25 // auto-format is off.
27 // However, Linux SPI-NAND driver expects the data returned as:
28 // +------+-----+
30 // +------+-----+
32 // So we assume all instructions matching the page_op template between ECC
34 // Here's how this spi-mem driver operates when reading:
35 // 1. Always set snf->autofmt = true in prepare_io_req (even when ECC is off).
37 // de-interleaved sector data and set FDM registers.
39 // +---------+---------+-----+------+------+-----+
41 // +---------+---------+-----+------+------+-----+
43 // read the data with auto-format off into the bounce buffer and copy
47 // As a limitation of this strategy, we won't be able to access any ECC parity
52 // as the bad block mark. After de-interleaving, this byte appears at [pagesize]
58 // 1. The [pagesize] byte on a nand page is used as BBM, which will appear at
59 // (page_size - (nsectors - 1) * spare_size) in the DMA buffer.
62 // We can't disagree with the BootROM, so after de-interleaving, we need to
64 // 1. Store the BBM at [page_size - (nsectors - 1) * spare_size] to [page_size],
66 // 2. Store the page data byte at [pagesize + (nsectors-1) * fdm] back to
67 // [page_size - (nsectors - 1) * spare_size]
77 #include <linux/dma-mapping.h>
80 #include <linux/mtd/nand-ecc-mtk.h>
82 #include <linux/spi/spi-mem.h>
83 #include <linux/mtd/nand.h>
281 struct mtk_ecc *ecc; member
290 static struct mtk_snand *nand_to_mtk_snand(struct nand_device *nand) in nand_to_mtk_snand() argument
292 struct nand_ecc_engine *eng = nand->ecc.engine; in nand_to_mtk_snand()
299 if (snf->buf_len >= size) in snand_prepare_bouncebuf()
301 kfree(snf->buf); in snand_prepare_bouncebuf()
302 snf->buf = kmalloc(size, GFP_KERNEL); in snand_prepare_bouncebuf()
303 if (!snf->buf) in snand_prepare_bouncebuf()
304 return -ENOMEM; in snand_prepare_bouncebuf()
305 snf->buf_len = size; in snand_prepare_bouncebuf()
306 memset(snf->buf, 0xff, snf->buf_len); in snand_prepare_bouncebuf()
312 return readl(snf->nfi_base + reg); in nfi_read32()
317 writel(val, snf->nfi_base + reg); in nfi_write32()
322 writew(val, snf->nfi_base + reg); in nfi_write16()
329 val = readl(snf->nfi_base + reg); in nfi_rmw32()
332 writel(val, snf->nfi_base + reg); in nfi_rmw32()
341 val = nfi_read32(snf, i & ~(es - 1)); in nfi_read_data()
354 ret = readw_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val, in mtk_nfi_reset()
355 !(val & snf->caps->mastersta_mask), 0, in mtk_nfi_reset()
358 dev_err(snf->dev, "NFI master is still busy after reset\n"); in mtk_nfi_reset()
362 ret = readl_poll_timeout(snf->nfi_base + NFI_STA, val, in mtk_nfi_reset()
366 dev_err(snf->dev, "Failed to reset NFI\n"); in mtk_nfi_reset()
370 fifo_mask = ((snf->caps->fifo_size - 1) << FIFO_RD_REMAIN_S) | in mtk_nfi_reset()
371 ((snf->caps->fifo_size - 1) << FIFO_WR_REMAIN_S); in mtk_nfi_reset()
372 ret = readw_poll_timeout(snf->nfi_base + NFI_FIFOSTA, val, in mtk_nfi_reset()
375 dev_err(snf->dev, "NFI FIFOs are not empty\n"); in mtk_nfi_reset()
389 ret = readl_poll_timeout(snf->nfi_base + SNF_STA_CTL1, val, in mtk_snand_mac_reset()
392 dev_err(snf->dev, "Failed to reset SNFI MAC\n"); in mtk_snand_mac_reset()
411 ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val, in mtk_snand_mac_trigger()
414 dev_err(snf->dev, "Timed out waiting for WIP_READY\n"); in mtk_snand_mac_trigger()
418 ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val, !(val & WIP), in mtk_snand_mac_trigger()
421 dev_err(snf->dev, "Timed out waiting for WIP cleared\n"); in mtk_snand_mac_trigger()
439 if (op->data.dir == SPI_MEM_DATA_IN) { in mtk_snand_mac_io()
440 rx_len = op->data.nbytes; in mtk_snand_mac_io()
441 rx_buf = op->data.buf.in; in mtk_snand_mac_io()
443 tx_buf = op->data.buf.out; in mtk_snand_mac_io()
448 for (i = 0; i < op->cmd.nbytes; i++, reg_offs++) { in mtk_snand_mac_io()
449 b = (op->cmd.opcode >> ((op->cmd.nbytes - i - 1) * 8)) & 0xff; in mtk_snand_mac_io()
452 nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val); in mtk_snand_mac_io()
457 for (i = 0; i < op->addr.nbytes; i++, reg_offs++) { in mtk_snand_mac_io()
458 b = (op->addr.val >> ((op->addr.nbytes - i - 1) * 8)) & 0xff; in mtk_snand_mac_io()
461 nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val); in mtk_snand_mac_io()
466 for (i = 0; i < op->dummy.nbytes; i++, reg_offs++) { in mtk_snand_mac_io()
468 nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val); in mtk_snand_mac_io()
473 if (op->data.dir == SPI_MEM_DATA_OUT) { in mtk_snand_mac_io()
474 for (i = 0; i < op->data.nbytes; i++, reg_offs++) { in mtk_snand_mac_io()
477 nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val); in mtk_snand_mac_io()
487 dev_dbg(snf->dev, "%d: %08X", i, in mtk_snand_mac_io()
490 dev_dbg(snf->dev, "SNF TX: %u RX: %u", reg_offs, rx_len); in mtk_snand_mac_io()
506 int spare_idx = -1; in mtk_snand_setup_pagefmt()
513 if (snf->nfi_cfg.page_size == page_size && in mtk_snand_setup_pagefmt()
514 snf->nfi_cfg.oob_size == oob_size) in mtk_snand_setup_pagefmt()
517 nsectors = page_size / snf->caps->sector_size; in mtk_snand_setup_pagefmt()
518 if (nsectors > snf->caps->max_sectors) { in mtk_snand_setup_pagefmt()
519 dev_err(snf->dev, "too many sectors required.\n"); in mtk_snand_setup_pagefmt()
523 if (snf->caps->sector_size == 512) { in mtk_snand_setup_pagefmt()
536 if (snf->caps->sector_size == 512) in mtk_snand_setup_pagefmt()
542 if (snf->caps->sector_size == 512) in mtk_snand_setup_pagefmt()
548 if (snf->caps->sector_size == 512) in mtk_snand_setup_pagefmt()
557 dev_err(snf->dev, "unsupported page size.\n"); in mtk_snand_setup_pagefmt()
564 if (snf->caps->sector_size == 1024) in mtk_snand_setup_pagefmt()
567 for (i = snf->caps->num_spare_size - 1; i >= 0; i--) { in mtk_snand_setup_pagefmt()
568 if (snf->caps->spare_sizes[i] <= spare_size) { in mtk_snand_setup_pagefmt()
569 spare_size = snf->caps->spare_sizes[i]; in mtk_snand_setup_pagefmt()
570 if (snf->caps->sector_size == 1024) in mtk_snand_setup_pagefmt()
578 dev_err(snf->dev, "unsupported spare size: %u\n", spare_size); in mtk_snand_setup_pagefmt()
583 (snf->caps->fdm_ecc_size << NFI_FDM_ECC_NUM_S) | in mtk_snand_setup_pagefmt()
584 (snf->caps->fdm_size << NFI_FDM_NUM_S) | in mtk_snand_setup_pagefmt()
589 snf->nfi_cfg.page_size = page_size; in mtk_snand_setup_pagefmt()
590 snf->nfi_cfg.oob_size = oob_size; in mtk_snand_setup_pagefmt()
591 snf->nfi_cfg.nsectors = nsectors; in mtk_snand_setup_pagefmt()
592 snf->nfi_cfg.spare_size = spare_size; in mtk_snand_setup_pagefmt()
594 dev_dbg(snf->dev, "page format: (%u + %u) * %u\n", in mtk_snand_setup_pagefmt()
595 snf->caps->sector_size, spare_size, nsectors); in mtk_snand_setup_pagefmt()
598 dev_err(snf->dev, "page size %u + %u is not supported\n", page_size, in mtk_snand_setup_pagefmt()
600 return -EOPNOTSUPP; in mtk_snand_setup_pagefmt()
606 // ECC area is not accessible in mtk_snand_ooblayout_ecc()
607 return -ERANGE; in mtk_snand_ooblayout_ecc()
613 struct nand_device *nand = mtd_to_nanddev(mtd); in mtk_snand_ooblayout_free() local
614 struct mtk_snand *ms = nand_to_mtk_snand(nand); in mtk_snand_ooblayout_free()
616 if (section >= ms->nfi_cfg.nsectors) in mtk_snand_ooblayout_free()
617 return -ERANGE; in mtk_snand_ooblayout_free()
619 oobfree->length = ms->caps->fdm_size - 1; in mtk_snand_ooblayout_free()
620 oobfree->offset = section * ms->caps->fdm_size + 1; in mtk_snand_ooblayout_free()
625 .ecc = mtk_snand_ooblayout_ecc,
629 static int mtk_snand_ecc_init_ctx(struct nand_device *nand) in mtk_snand_ecc_init_ctx() argument
631 struct mtk_snand *snf = nand_to_mtk_snand(nand); in mtk_snand_ecc_init_ctx()
632 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; in mtk_snand_ecc_init_ctx()
633 struct nand_ecc_props *reqs = &nand->ecc.requirements; in mtk_snand_ecc_init_ctx()
634 struct nand_ecc_props *user = &nand->ecc.user_conf; in mtk_snand_ecc_init_ctx()
635 struct mtd_info *mtd = nanddev_to_mtd(nand); in mtk_snand_ecc_init_ctx()
636 int step_size = 0, strength = 0, desired_correction = 0, steps; in mtk_snand_ecc_init_ctx() local
642 ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize, in mtk_snand_ecc_init_ctx()
643 nand->memorg.oobsize); in mtk_snand_ecc_init_ctx()
649 return -ENOMEM; in mtk_snand_ecc_init_ctx()
651 nand->ecc.ctx.priv = ecc_cfg; in mtk_snand_ecc_init_ctx()
653 if (user->step_size && user->strength) { in mtk_snand_ecc_init_ctx()
654 step_size = user->step_size; in mtk_snand_ecc_init_ctx()
655 strength = user->strength; in mtk_snand_ecc_init_ctx()
657 } else if (reqs->step_size && reqs->strength) { in mtk_snand_ecc_init_ctx()
658 step_size = reqs->step_size; in mtk_snand_ecc_init_ctx()
659 strength = reqs->strength; in mtk_snand_ecc_init_ctx()
662 if (step_size && strength) { in mtk_snand_ecc_init_ctx()
663 steps = mtd->writesize / step_size; in mtk_snand_ecc_init_ctx()
664 desired_correction = steps * strength; in mtk_snand_ecc_init_ctx()
665 strength = desired_correction / snf->nfi_cfg.nsectors; in mtk_snand_ecc_init_ctx()
668 ecc_cfg->mode = ECC_NFI_MODE; in mtk_snand_ecc_init_ctx()
669 ecc_cfg->sectors = snf->nfi_cfg.nsectors; in mtk_snand_ecc_init_ctx()
670 ecc_cfg->len = snf->caps->sector_size + snf->caps->fdm_ecc_size; in mtk_snand_ecc_init_ctx()
672 // calculate the max possible strength under current page format in mtk_snand_ecc_init_ctx()
673 parity_bits = mtk_ecc_get_parity_bits(snf->ecc); in mtk_snand_ecc_init_ctx()
674 max_ecc_bytes = snf->nfi_cfg.spare_size - snf->caps->fdm_size; in mtk_snand_ecc_init_ctx()
675 ecc_cfg->strength = max_ecc_bytes * 8 / parity_bits; in mtk_snand_ecc_init_ctx()
676 mtk_ecc_adjust_strength(snf->ecc, &ecc_cfg->strength); in mtk_snand_ecc_init_ctx()
678 // if there's a user requested strength, find the minimum strength that in mtk_snand_ecc_init_ctx()
679 // meets the requirement. Otherwise use the maximum strength which is in mtk_snand_ecc_init_ctx()
681 if (ecc_user && strength) { in mtk_snand_ecc_init_ctx()
682 u32 s_next = ecc_cfg->strength - 1; in mtk_snand_ecc_init_ctx()
685 mtk_ecc_adjust_strength(snf->ecc, &s_next); in mtk_snand_ecc_init_ctx()
686 if (s_next >= ecc_cfg->strength) in mtk_snand_ecc_init_ctx()
688 if (s_next < strength) in mtk_snand_ecc_init_ctx()
690 s_next = ecc_cfg->strength - 1; in mtk_snand_ecc_init_ctx()
696 conf->step_size = snf->caps->sector_size; in mtk_snand_ecc_init_ctx()
697 conf->strength = ecc_cfg->strength; in mtk_snand_ecc_init_ctx()
699 if (ecc_cfg->strength < strength) in mtk_snand_ecc_init_ctx()
700 dev_warn(snf->dev, "unable to fulfill ECC of %u bits.\n", in mtk_snand_ecc_init_ctx()
701 strength); in mtk_snand_ecc_init_ctx()
702 dev_info(snf->dev, "ECC strength: %u bits per %u bytes\n", in mtk_snand_ecc_init_ctx()
703 ecc_cfg->strength, snf->caps->sector_size); in mtk_snand_ecc_init_ctx()
708 static void mtk_snand_ecc_cleanup_ctx(struct nand_device *nand) in mtk_snand_ecc_cleanup_ctx() argument
710 struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand); in mtk_snand_ecc_cleanup_ctx()
715 static int mtk_snand_ecc_prepare_io_req(struct nand_device *nand, in mtk_snand_ecc_prepare_io_req() argument
718 struct mtk_snand *snf = nand_to_mtk_snand(nand); in mtk_snand_ecc_prepare_io_req()
719 struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand); in mtk_snand_ecc_prepare_io_req()
722 ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize, in mtk_snand_ecc_prepare_io_req()
723 nand->memorg.oobsize); in mtk_snand_ecc_prepare_io_req()
726 snf->autofmt = true; in mtk_snand_ecc_prepare_io_req()
727 snf->ecc_cfg = ecc_cfg; in mtk_snand_ecc_prepare_io_req()
731 static int mtk_snand_ecc_finish_io_req(struct nand_device *nand, in mtk_snand_ecc_finish_io_req() argument
734 struct mtk_snand *snf = nand_to_mtk_snand(nand); in mtk_snand_ecc_finish_io_req()
735 struct mtd_info *mtd = nanddev_to_mtd(nand); in mtk_snand_ecc_finish_io_req()
737 snf->ecc_cfg = NULL; in mtk_snand_ecc_finish_io_req()
738 snf->autofmt = false; in mtk_snand_ecc_finish_io_req()
739 if ((req->mode == MTD_OPS_RAW) || (req->type != NAND_PAGE_READ)) in mtk_snand_ecc_finish_io_req()
742 if (snf->ecc_stats.failed) in mtk_snand_ecc_finish_io_req()
743 mtd->ecc_stats.failed += snf->ecc_stats.failed; in mtk_snand_ecc_finish_io_req()
744 mtd->ecc_stats.corrected += snf->ecc_stats.corrected; in mtk_snand_ecc_finish_io_req()
745 return snf->ecc_stats.failed ? -EBADMSG : snf->ecc_stats.bitflips; in mtk_snand_ecc_finish_io_req()
761 for (i = 0; i < snf->nfi_cfg.nsectors; i++) { in mtk_snand_read_fdm()
765 for (j = 0; j < snf->caps->fdm_size; j++) in mtk_snand_read_fdm()
768 oobptr += snf->caps->fdm_size; in mtk_snand_read_fdm()
774 u32 fdm_size = snf->caps->fdm_size; in mtk_snand_write_fdm()
779 for (i = 0; i < snf->nfi_cfg.nsectors; i++) { in mtk_snand_write_fdm()
789 << ((j - 4) * 8); in mtk_snand_write_fdm()
803 if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1) in mtk_snand_bm_swap()
806 // swap [pagesize] byte on nand with the first fdm byte in mtk_snand_bm_swap()
808 buf_bbm_pos = snf->nfi_cfg.page_size - in mtk_snand_bm_swap()
809 (snf->nfi_cfg.nsectors - 1) * snf->nfi_cfg.spare_size; in mtk_snand_bm_swap()
810 fdm_bbm_pos = snf->nfi_cfg.page_size + in mtk_snand_bm_swap()
811 (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size; in mtk_snand_bm_swap()
813 swap(snf->buf[fdm_bbm_pos], buf[buf_bbm_pos]); in mtk_snand_bm_swap()
820 if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1) in mtk_snand_fdm_bm_swap()
824 fdm_bbm_pos1 = snf->nfi_cfg.page_size; in mtk_snand_fdm_bm_swap()
825 fdm_bbm_pos2 = snf->nfi_cfg.page_size + in mtk_snand_fdm_bm_swap()
826 (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size; in mtk_snand_fdm_bm_swap()
827 swap(snf->buf[fdm_bbm_pos1], snf->buf[fdm_bbm_pos2]); in mtk_snand_fdm_bm_swap()
833 u8 *buf = snf->buf; in mtk_snand_read_page_cache()
834 u8 *buf_fdm = buf + snf->nfi_cfg.page_size; in mtk_snand_read_page_cache()
836 u32 op_addr = op->addr.val; in mtk_snand_read_page_cache()
839 u32 dummy_clk = (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth); in mtk_snand_read_page_cache()
841 u32 dma_len = snf->buf_len; in mtk_snand_read_page_cache()
846 if (snf->autofmt) { in mtk_snand_read_page_cache()
850 dma_len = snf->nfi_cfg.page_size; in mtk_snand_read_page_cache()
852 if (op->data.ecc) in mtk_snand_read_page_cache()
856 // Bits higher than that in op->addr are kept and sent over SPI in mtk_snand_read_page_cache()
859 last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size); in mtk_snand_read_page_cache()
860 mask = (1 << last_bit) - 1; in mtk_snand_read_page_cache()
865 if (rd_offset == 0 && op->data.nbytes >= snf->nfi_cfg.page_size) in mtk_snand_read_page_cache()
866 buf = op->data.buf.in; in mtk_snand_read_page_cache()
874 (op->cmd.opcode << DATA_READ_CMD_S)); in mtk_snand_read_page_cache()
880 if (op->data.buswidth == 4) in mtk_snand_read_page_cache()
881 rd_mode = op->addr.buswidth == 4 ? DATA_READ_MODE_QUAD : in mtk_snand_read_page_cache()
883 else if (op->data.buswidth == 2) in mtk_snand_read_page_cache()
884 rd_mode = op->addr.buswidth == 2 ? DATA_READ_MODE_DUAL : in mtk_snand_read_page_cache()
893 rd_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) * in mtk_snand_read_page_cache()
894 snf->nfi_cfg.nsectors; in mtk_snand_read_page_cache()
903 nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S)); in mtk_snand_read_page_cache()
905 buf_dma = dma_map_single(snf->dev, buf, dma_len, DMA_FROM_DEVICE); in mtk_snand_read_page_cache()
906 ret = dma_mapping_error(snf->dev, buf_dma); in mtk_snand_read_page_cache()
908 dev_err(snf->dev, "DMA mapping failed.\n"); in mtk_snand_read_page_cache()
912 if (op->data.ecc) { in mtk_snand_read_page_cache()
913 snf->ecc_cfg->op = ECC_DECODE; in mtk_snand_read_page_cache()
914 ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg); in mtk_snand_read_page_cache()
920 reinit_completion(&snf->op_done); in mtk_snand_read_page_cache()
930 &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) { in mtk_snand_read_page_cache()
931 dev_err(snf->dev, "DMA timed out for reading from cache.\n"); in mtk_snand_read_page_cache()
932 ret = -ETIMEDOUT; in mtk_snand_read_page_cache()
937 ret = readl_poll_timeout(snf->nfi_base + NFI_BYTELEN, val, in mtk_snand_read_page_cache()
938 BUS_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0, in mtk_snand_read_page_cache()
941 dev_err(snf->dev, "Timed out waiting for BUS_SEC_CNTR\n"); in mtk_snand_read_page_cache()
946 ret = readl_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val, in mtk_snand_read_page_cache()
947 !(val & snf->caps->mastersta_mask), 0, in mtk_snand_read_page_cache()
950 dev_err(snf->dev, "Timed out waiting for bus becoming idle\n"); in mtk_snand_read_page_cache()
954 if (op->data.ecc) { in mtk_snand_read_page_cache()
955 ret = mtk_ecc_wait_done(snf->ecc, ECC_DECODE); in mtk_snand_read_page_cache()
957 dev_err(snf->dev, "wait ecc done timeout\n"); in mtk_snand_read_page_cache()
960 // save status before disabling ecc in mtk_snand_read_page_cache()
961 mtk_ecc_get_stats(snf->ecc, &snf->ecc_stats, in mtk_snand_read_page_cache()
962 snf->nfi_cfg.nsectors); in mtk_snand_read_page_cache()
965 dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE); in mtk_snand_read_page_cache()
967 if (snf->autofmt) { in mtk_snand_read_page_cache()
969 if (snf->caps->bbm_swap) { in mtk_snand_read_page_cache()
977 memset(op->data.buf.in, 0xff, op->data.nbytes); in mtk_snand_read_page_cache()
978 snf->ecc_stats.bitflips = 0; in mtk_snand_read_page_cache()
979 snf->ecc_stats.failed = 0; in mtk_snand_read_page_cache()
980 snf->ecc_stats.corrected = 0; in mtk_snand_read_page_cache()
982 if (buf == op->data.buf.in) { in mtk_snand_read_page_cache()
983 u32 cap_len = snf->buf_len - snf->nfi_cfg.page_size; in mtk_snand_read_page_cache()
984 u32 req_left = op->data.nbytes - snf->nfi_cfg.page_size; in mtk_snand_read_page_cache()
987 memcpy(op->data.buf.in + snf->nfi_cfg.page_size, in mtk_snand_read_page_cache()
990 } else if (rd_offset < snf->buf_len) { in mtk_snand_read_page_cache()
991 u32 cap_len = snf->buf_len - rd_offset; in mtk_snand_read_page_cache()
993 if (op->data.nbytes < cap_len) in mtk_snand_read_page_cache()
994 cap_len = op->data.nbytes; in mtk_snand_read_page_cache()
995 memcpy(op->data.buf.in, snf->buf + rd_offset, cap_len); in mtk_snand_read_page_cache()
999 if (op->data.ecc) in mtk_snand_read_page_cache()
1000 mtk_ecc_disable(snf->ecc); in mtk_snand_read_page_cache()
1005 dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE); in mtk_snand_read_page_cache()
1027 u32 op_addr = op->addr.val; in mtk_snand_write_page_cache()
1033 u32 dma_len = snf->buf_len; in mtk_snand_write_page_cache()
1038 if (snf->autofmt) { in mtk_snand_write_page_cache()
1042 dma_len = snf->nfi_cfg.page_size; in mtk_snand_write_page_cache()
1044 if (op->data.ecc) in mtk_snand_write_page_cache()
1047 last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size); in mtk_snand_write_page_cache()
1048 mask = (1 << last_bit) - 1; in mtk_snand_write_page_cache()
1056 memset(snf->buf, 0xff, wr_offset); in mtk_snand_write_page_cache()
1058 cap_len = snf->buf_len - wr_offset; in mtk_snand_write_page_cache()
1059 if (op->data.nbytes < cap_len) in mtk_snand_write_page_cache()
1060 cap_len = op->data.nbytes; in mtk_snand_write_page_cache()
1061 memcpy(snf->buf + wr_offset, op->data.buf.out, cap_len); in mtk_snand_write_page_cache()
1062 if (snf->autofmt) { in mtk_snand_write_page_cache()
1063 if (snf->caps->bbm_swap) { in mtk_snand_write_page_cache()
1065 mtk_snand_bm_swap(snf, snf->buf); in mtk_snand_write_page_cache()
1067 mtk_snand_write_fdm(snf, snf->buf + snf->nfi_cfg.page_size); in mtk_snand_write_page_cache()
1071 nfi_write32(snf, SNF_PG_CTL1, (op->cmd.opcode << PG_LOAD_CMD_S)); in mtk_snand_write_page_cache()
1077 if (op->data.buswidth == 4) in mtk_snand_write_page_cache()
1084 wr_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) * in mtk_snand_write_page_cache()
1085 snf->nfi_cfg.nsectors; in mtk_snand_write_page_cache()
1094 nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S)); in mtk_snand_write_page_cache()
1095 buf_dma = dma_map_single(snf->dev, snf->buf, dma_len, DMA_TO_DEVICE); in mtk_snand_write_page_cache()
1096 ret = dma_mapping_error(snf->dev, buf_dma); in mtk_snand_write_page_cache()
1098 dev_err(snf->dev, "DMA mapping failed.\n"); in mtk_snand_write_page_cache()
1102 if (op->data.ecc) { in mtk_snand_write_page_cache()
1103 snf->ecc_cfg->op = ECC_ENCODE; in mtk_snand_write_page_cache()
1104 ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg); in mtk_snand_write_page_cache()
1110 reinit_completion(&snf->op_done); in mtk_snand_write_page_cache()
1121 &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) { in mtk_snand_write_page_cache()
1122 dev_err(snf->dev, "DMA timed out for program load.\n"); in mtk_snand_write_page_cache()
1123 ret = -ETIMEDOUT; in mtk_snand_write_page_cache()
1128 ret = readl_poll_timeout(snf->nfi_base + NFI_ADDRCNTR, val, in mtk_snand_write_page_cache()
1129 NFI_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0, in mtk_snand_write_page_cache()
1132 dev_err(snf->dev, "Timed out waiting for NFI_SEC_CNTR\n"); in mtk_snand_write_page_cache()
1135 if (op->data.ecc) in mtk_snand_write_page_cache()
1136 mtk_ecc_disable(snf->ecc); in mtk_snand_write_page_cache()
1138 dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_TO_DEVICE); in mtk_snand_write_page_cache()
1158 * mtk_snand_is_page_ops() - check if the op is a controller supported page op.
1159 * @op spi-mem op to check
1164 * instructions found on SPI-NAND with 2-byte address.
1171 if (op->addr.nbytes != 2) in mtk_snand_is_page_ops()
1174 if (op->addr.buswidth != 1 && op->addr.buswidth != 2 && in mtk_snand_is_page_ops()
1175 op->addr.buswidth != 4) in mtk_snand_is_page_ops()
1179 if (op->data.dir == SPI_MEM_DATA_IN) { in mtk_snand_is_page_ops()
1181 if (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth > in mtk_snand_is_page_ops()
1185 if ((op->addr.buswidth == 4 || op->addr.buswidth == 1) && in mtk_snand_is_page_ops()
1186 op->data.buswidth == 4) in mtk_snand_is_page_ops()
1190 if ((op->addr.buswidth == 2 || op->addr.buswidth == 1) && in mtk_snand_is_page_ops()
1191 op->data.buswidth == 2) in mtk_snand_is_page_ops()
1195 if (op->addr.buswidth == 1 && op->data.buswidth == 1) in mtk_snand_is_page_ops()
1197 } else if (op->data.dir == SPI_MEM_DATA_OUT) { in mtk_snand_is_page_ops()
1199 if (op->dummy.nbytes) in mtk_snand_is_page_ops()
1202 if (op->addr.buswidth == 1 && op->data.buswidth == 4) in mtk_snand_is_page_ops()
1205 if (op->addr.buswidth == 1 && op->data.buswidth == 1) in mtk_snand_is_page_ops()
1216 if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1) in mtk_snand_supports_op()
1220 return ((op->addr.nbytes == 0 || op->addr.buswidth == 1) && in mtk_snand_supports_op()
1221 (op->dummy.nbytes == 0 || op->dummy.buswidth == 1) && in mtk_snand_supports_op()
1222 (op->data.nbytes == 0 || op->data.buswidth == 1)); in mtk_snand_supports_op()
1227 struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master); in mtk_snand_adjust_op_size()
1235 if (ms->autofmt) in mtk_snand_adjust_op_size()
1237 l = ms->caps->sector_size + ms->nfi_cfg.spare_size; in mtk_snand_adjust_op_size()
1238 l *= ms->nfi_cfg.nsectors; in mtk_snand_adjust_op_size()
1239 if (op->data.nbytes > l) in mtk_snand_adjust_op_size()
1240 op->data.nbytes = l; in mtk_snand_adjust_op_size()
1242 size_t hl = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; in mtk_snand_adjust_op_size()
1245 return -EOPNOTSUPP; in mtk_snand_adjust_op_size()
1246 if (op->data.nbytes > SNF_GPRAM_SIZE - hl) in mtk_snand_adjust_op_size()
1247 op->data.nbytes = SNF_GPRAM_SIZE - hl; in mtk_snand_adjust_op_size()
1254 struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master); in mtk_snand_exec_op()
1256 dev_dbg(ms->dev, "OP %02x ADDR %08llX@%d:%u DATA %d:%u", op->cmd.opcode, in mtk_snand_exec_op()
1257 op->addr.val, op->addr.buswidth, op->addr.nbytes, in mtk_snand_exec_op()
1258 op->data.buswidth, op->data.nbytes); in mtk_snand_exec_op()
1260 if (op->data.dir == SPI_MEM_DATA_IN) in mtk_snand_exec_op()
1276 .ecc = true,
1291 complete(&snf->op_done); in mtk_snand_irq()
1296 { .compatible = "mediatek,mt7622-snand", .data = &mt7622_snand_caps },
1297 { .compatible = "mediatek,mt7629-snand", .data = &mt7629_snand_caps },
1307 ret = clk_prepare_enable(ms->nfi_clk); in mtk_snand_enable_clk()
1309 dev_err(ms->dev, "unable to enable nfi clk\n"); in mtk_snand_enable_clk()
1312 ret = clk_prepare_enable(ms->pad_clk); in mtk_snand_enable_clk()
1314 dev_err(ms->dev, "unable to enable pad clk\n"); in mtk_snand_enable_clk()
1319 clk_disable_unprepare(ms->nfi_clk); in mtk_snand_enable_clk()
1325 clk_disable_unprepare(ms->pad_clk); in mtk_snand_disable_clk()
1326 clk_disable_unprepare(ms->nfi_clk); in mtk_snand_disable_clk()
1331 struct device_node *np = pdev->dev.of_node; in mtk_snand_probe()
1339 return -EINVAL; in mtk_snand_probe()
1341 ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*ms)); in mtk_snand_probe()
1343 return -ENOMEM; in mtk_snand_probe()
1348 ms->ctlr = ctlr; in mtk_snand_probe()
1349 ms->caps = dev_id->data; in mtk_snand_probe()
1351 ms->ecc = of_mtk_ecc_get(np); in mtk_snand_probe()
1352 if (IS_ERR(ms->ecc)) in mtk_snand_probe()
1353 return PTR_ERR(ms->ecc); in mtk_snand_probe()
1354 else if (!ms->ecc) in mtk_snand_probe()
1355 return -ENODEV; in mtk_snand_probe()
1357 ms->nfi_base = devm_platform_ioremap_resource(pdev, 0); in mtk_snand_probe()
1358 if (IS_ERR(ms->nfi_base)) { in mtk_snand_probe()
1359 ret = PTR_ERR(ms->nfi_base); in mtk_snand_probe()
1363 ms->dev = &pdev->dev; in mtk_snand_probe()
1365 ms->nfi_clk = devm_clk_get(&pdev->dev, "nfi_clk"); in mtk_snand_probe()
1366 if (IS_ERR(ms->nfi_clk)) { in mtk_snand_probe()
1367 ret = PTR_ERR(ms->nfi_clk); in mtk_snand_probe()
1368 dev_err(&pdev->dev, "unable to get nfi_clk, err = %d\n", ret); in mtk_snand_probe()
1372 ms->pad_clk = devm_clk_get(&pdev->dev, "pad_clk"); in mtk_snand_probe()
1373 if (IS_ERR(ms->pad_clk)) { in mtk_snand_probe()
1374 ret = PTR_ERR(ms->pad_clk); in mtk_snand_probe()
1375 dev_err(&pdev->dev, "unable to get pad_clk, err = %d\n", ret); in mtk_snand_probe()
1383 init_completion(&ms->op_done); in mtk_snand_probe()
1385 ms->irq = platform_get_irq(pdev, 0); in mtk_snand_probe()
1386 if (ms->irq < 0) { in mtk_snand_probe()
1387 ret = ms->irq; in mtk_snand_probe()
1390 ret = devm_request_irq(ms->dev, ms->irq, mtk_snand_irq, 0x0, in mtk_snand_probe()
1391 "mtk-snand", ms); in mtk_snand_probe()
1393 dev_err(ms->dev, "failed to request snfi irq\n"); in mtk_snand_probe()
1397 ret = dma_set_mask(ms->dev, DMA_BIT_MASK(32)); in mtk_snand_probe()
1399 dev_err(ms->dev, "failed to set dma mask\n"); in mtk_snand_probe()
1407 // before ECC is called. in mtk_snand_probe()
1408 ret = mtk_snand_setup_pagefmt(ms, ms->caps->sector_size, in mtk_snand_probe()
1409 ms->caps->spare_sizes[0]); in mtk_snand_probe()
1411 dev_err(ms->dev, "failed to set initial page format\n"); in mtk_snand_probe()
1415 // setup ECC engine in mtk_snand_probe()
1416 ms->ecc_eng.dev = &pdev->dev; in mtk_snand_probe()
1417 ms->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED; in mtk_snand_probe()
1418 ms->ecc_eng.ops = &mtk_snfi_ecc_engine_ops; in mtk_snand_probe()
1419 ms->ecc_eng.priv = ms; in mtk_snand_probe()
1421 ret = nand_ecc_register_on_host_hw_engine(&ms->ecc_eng); in mtk_snand_probe()
1423 dev_err(&pdev->dev, "failed to register ecc engine.\n"); in mtk_snand_probe()
1427 ctlr->num_chipselect = 1; in mtk_snand_probe()
1428 ctlr->mem_ops = &mtk_snand_mem_ops; in mtk_snand_probe()
1429 ctlr->mem_caps = &mtk_snand_mem_caps; in mtk_snand_probe()
1430 ctlr->bits_per_word_mask = SPI_BPW_MASK(8); in mtk_snand_probe()
1431 ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD; in mtk_snand_probe()
1432 ctlr->dev.of_node = pdev->dev.of_node; in mtk_snand_probe()
1435 dev_err(&pdev->dev, "spi_register_controller failed.\n"); in mtk_snand_probe()
1443 mtk_ecc_release(ms->ecc); in mtk_snand_probe()
1454 mtk_ecc_release(ms->ecc); in mtk_snand_remove()
1455 kfree(ms->buf); in mtk_snand_remove()
1463 .name = "mtk-snand",
1472 MODULE_DESCRIPTION("MeidaTek SPI-NAND Flash Controller Driver");