1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/mtd/mtd.h>
33 #include <linux/mtd/rawnand.h>
34 #include <linux/mtd/partitions.h>
35 #include <linux/clk.h>
36 #include <linux/delay.h>
37 #include <linux/dmaengine.h>
38 #include <linux/interrupt.h>
39 #include <linux/iopoll.h>
40 #include <linux/reset.h>
41
42 #define NFC_REG_CTL 0x0000
43 #define NFC_REG_ST 0x0004
44 #define NFC_REG_INT 0x0008
45 #define NFC_REG_TIMING_CTL 0x000C
46 #define NFC_REG_TIMING_CFG 0x0010
47 #define NFC_REG_ADDR_LOW 0x0014
48 #define NFC_REG_ADDR_HIGH 0x0018
49 #define NFC_REG_SECTOR_NUM 0x001C
50 #define NFC_REG_CNT 0x0020
51 #define NFC_REG_CMD 0x0024
52 #define NFC_REG_RCMD_SET 0x0028
53 #define NFC_REG_WCMD_SET 0x002C
54 #define NFC_REG_IO_DATA 0x0030
55 #define NFC_REG_ECC_CTL 0x0034
56 #define NFC_REG_ECC_ST 0x0038
57 #define NFC_REG_DEBUG 0x003C
58 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
59 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
60 #define NFC_REG_SPARE_AREA 0x00A0
61 #define NFC_REG_PAT_ID 0x00A4
62 #define NFC_RAM0_BASE 0x0400
63 #define NFC_RAM1_BASE 0x0800
64
65 /* define bit use in NFC_CTL */
66 #define NFC_EN BIT(0)
67 #define NFC_RESET BIT(1)
68 #define NFC_BUS_WIDTH_MSK BIT(2)
69 #define NFC_BUS_WIDTH_8 (0 << 2)
70 #define NFC_BUS_WIDTH_16 (1 << 2)
71 #define NFC_RB_SEL_MSK BIT(3)
72 #define NFC_RB_SEL(x) ((x) << 3)
73 #define NFC_CE_SEL_MSK GENMASK(26, 24)
74 #define NFC_CE_SEL(x) ((x) << 24)
75 #define NFC_CE_CTL BIT(6)
76 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
77 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
78 #define NFC_SAM BIT(12)
79 #define NFC_RAM_METHOD BIT(14)
80 #define NFC_DEBUG_CTL BIT(31)
81
82 /* define bit use in NFC_ST */
83 #define NFC_RB_B2R BIT(0)
84 #define NFC_CMD_INT_FLAG BIT(1)
85 #define NFC_DMA_INT_FLAG BIT(2)
86 #define NFC_CMD_FIFO_STATUS BIT(3)
87 #define NFC_STA BIT(4)
88 #define NFC_NATCH_INT_FLAG BIT(5)
89 #define NFC_RB_STATE(x) BIT(x + 8)
90
91 /* define bit use in NFC_INT */
92 #define NFC_B2R_INT_ENABLE BIT(0)
93 #define NFC_CMD_INT_ENABLE BIT(1)
94 #define NFC_DMA_INT_ENABLE BIT(2)
95 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
96 NFC_CMD_INT_ENABLE | \
97 NFC_DMA_INT_ENABLE)
98
99 /* define bit use in NFC_TIMING_CTL */
100 #define NFC_TIMING_CTL_EDO BIT(8)
101
102 /* define NFC_TIMING_CFG register layout */
103 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
104 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
105 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
106 (((tCAD) & 0x7) << 8))
107
108 /* define bit use in NFC_CMD */
109 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
110 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
111 #define NFC_CMD(x) (x)
112 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
113 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
114 #define NFC_SEND_ADR BIT(19)
115 #define NFC_ACCESS_DIR BIT(20)
116 #define NFC_DATA_TRANS BIT(21)
117 #define NFC_SEND_CMD1 BIT(22)
118 #define NFC_WAIT_FLAG BIT(23)
119 #define NFC_SEND_CMD2 BIT(24)
120 #define NFC_SEQ BIT(25)
121 #define NFC_DATA_SWAP_METHOD BIT(26)
122 #define NFC_ROW_AUTO_INC BIT(27)
123 #define NFC_SEND_CMD3 BIT(28)
124 #define NFC_SEND_CMD4 BIT(29)
125 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
126 #define NFC_NORMAL_OP (0 << 30)
127 #define NFC_ECC_OP (1 << 30)
128 #define NFC_PAGE_OP (2U << 30)
129
130 /* define bit use in NFC_RCMD_SET */
131 #define NFC_READ_CMD_MSK GENMASK(7, 0)
132 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
133 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
134
135 /* define bit use in NFC_WCMD_SET */
136 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
137 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
138 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
139 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
140
141 /* define bit use in NFC_ECC_CTL */
142 #define NFC_ECC_EN BIT(0)
143 #define NFC_ECC_PIPELINE BIT(3)
144 #define NFC_ECC_EXCEPTION BIT(4)
145 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
146 #define NFC_ECC_BLOCK_512 BIT(5)
147 #define NFC_RANDOM_EN BIT(9)
148 #define NFC_RANDOM_DIRECTION BIT(10)
149 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
150 #define NFC_ECC_MODE(x) ((x) << 12)
151 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
152 #define NFC_RANDOM_SEED(x) ((x) << 16)
153
154 /* define bit use in NFC_ECC_ST */
155 #define NFC_ECC_ERR(x) BIT(x)
156 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
157 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
158 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
159
160 #define NFC_DEFAULT_TIMEOUT_MS 1000
161
162 #define NFC_SRAM_SIZE 1024
163
164 #define NFC_MAX_CS 7
165
166 /*
167 * Chip Select structure: stores information related to NAND Chip Select
168 *
169 * @cs: the NAND CS id used to communicate with a NAND Chip
170 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the
171 * NFC
172 */
173 struct sunxi_nand_chip_sel {
174 u8 cs;
175 s8 rb;
176 };
177
178 /*
179 * sunxi HW ECC infos: stores information related to HW ECC support
180 *
181 * @mode: the sunxi ECC mode field deduced from ECC requirements
182 */
183 struct sunxi_nand_hw_ecc {
184 int mode;
185 };
186
187 /*
188 * NAND chip structure: stores NAND chip device related information
189 *
190 * @node: used to store NAND chips into a list
191 * @nand: base NAND chip structure
192 * @mtd: base MTD structure
193 * @clk_rate: clk_rate required for this NAND chip
194 * @timing_cfg TIMING_CFG register value for this NAND chip
195 * @selected: current active CS
196 * @nsels: number of CS lines required by the NAND chip
197 * @sels: array of CS lines descriptions
198 */
199 struct sunxi_nand_chip {
200 struct list_head node;
201 struct nand_chip nand;
202 unsigned long clk_rate;
203 u32 timing_cfg;
204 u32 timing_ctl;
205 int selected;
206 int addr_cycles;
207 u32 addr[2];
208 int cmd_cycles;
209 u8 cmd[2];
210 int nsels;
211 struct sunxi_nand_chip_sel sels[0];
212 };
213
to_sunxi_nand(struct nand_chip * nand)214 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
215 {
216 return container_of(nand, struct sunxi_nand_chip, nand);
217 }
218
219 /*
220 * NAND Controller structure: stores sunxi NAND controller information
221 *
222 * @controller: base controller structure
223 * @dev: parent device (used to print error messages)
224 * @regs: NAND controller registers
225 * @ahb_clk: NAND Controller AHB clock
226 * @mod_clk: NAND Controller mod clock
227 * @assigned_cs: bitmask describing already assigned CS lines
228 * @clk_rate: NAND controller current clock rate
229 * @chips: a list containing all the NAND chips attached to
230 * this NAND controller
231 * @complete: a completion object used to wait for NAND
232 * controller events
233 */
234 struct sunxi_nfc {
235 struct nand_controller controller;
236 struct device *dev;
237 void __iomem *regs;
238 struct clk *ahb_clk;
239 struct clk *mod_clk;
240 struct reset_control *reset;
241 unsigned long assigned_cs;
242 unsigned long clk_rate;
243 struct list_head chips;
244 struct completion complete;
245 struct dma_chan *dmac;
246 };
247
to_sunxi_nfc(struct nand_controller * ctrl)248 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
249 {
250 return container_of(ctrl, struct sunxi_nfc, controller);
251 }
252
sunxi_nfc_interrupt(int irq,void * dev_id)253 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
254 {
255 struct sunxi_nfc *nfc = dev_id;
256 u32 st = readl(nfc->regs + NFC_REG_ST);
257 u32 ien = readl(nfc->regs + NFC_REG_INT);
258
259 if (!(ien & st))
260 return IRQ_NONE;
261
262 if ((ien & st) == ien)
263 complete(&nfc->complete);
264
265 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
266 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
267
268 return IRQ_HANDLED;
269 }
270
sunxi_nfc_wait_events(struct sunxi_nfc * nfc,u32 events,bool use_polling,unsigned int timeout_ms)271 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
272 bool use_polling, unsigned int timeout_ms)
273 {
274 int ret;
275
276 if (events & ~NFC_INT_MASK)
277 return -EINVAL;
278
279 if (!timeout_ms)
280 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
281
282 if (!use_polling) {
283 init_completion(&nfc->complete);
284
285 writel(events, nfc->regs + NFC_REG_INT);
286
287 ret = wait_for_completion_timeout(&nfc->complete,
288 msecs_to_jiffies(timeout_ms));
289 if (!ret)
290 ret = -ETIMEDOUT;
291 else
292 ret = 0;
293
294 writel(0, nfc->regs + NFC_REG_INT);
295 } else {
296 u32 status;
297
298 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
299 (status & events) == events, 1,
300 timeout_ms * 1000);
301 }
302
303 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
304
305 if (ret)
306 dev_err(nfc->dev, "wait interrupt timedout\n");
307
308 return ret;
309 }
310
sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc * nfc)311 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
312 {
313 u32 status;
314 int ret;
315
316 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
317 !(status & NFC_CMD_FIFO_STATUS), 1,
318 NFC_DEFAULT_TIMEOUT_MS * 1000);
319 if (ret)
320 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
321
322 return ret;
323 }
324
sunxi_nfc_rst(struct sunxi_nfc * nfc)325 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
326 {
327 u32 ctl;
328 int ret;
329
330 writel(0, nfc->regs + NFC_REG_ECC_CTL);
331 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
332
333 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
334 !(ctl & NFC_RESET), 1,
335 NFC_DEFAULT_TIMEOUT_MS * 1000);
336 if (ret)
337 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
338
339 return ret;
340 }
341
sunxi_nfc_dma_op_prepare(struct mtd_info * mtd,const void * buf,int chunksize,int nchunks,enum dma_data_direction ddir,struct scatterlist * sg)342 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
343 int chunksize, int nchunks,
344 enum dma_data_direction ddir,
345 struct scatterlist *sg)
346 {
347 struct nand_chip *nand = mtd_to_nand(mtd);
348 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
349 struct dma_async_tx_descriptor *dmad;
350 enum dma_transfer_direction tdir;
351 dma_cookie_t dmat;
352 int ret;
353
354 if (ddir == DMA_FROM_DEVICE)
355 tdir = DMA_DEV_TO_MEM;
356 else
357 tdir = DMA_MEM_TO_DEV;
358
359 sg_init_one(sg, buf, nchunks * chunksize);
360 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
361 if (!ret)
362 return -ENOMEM;
363
364 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
365 if (!dmad) {
366 ret = -EINVAL;
367 goto err_unmap_buf;
368 }
369
370 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
371 nfc->regs + NFC_REG_CTL);
372 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
373 writel(chunksize, nfc->regs + NFC_REG_CNT);
374 dmat = dmaengine_submit(dmad);
375
376 ret = dma_submit_error(dmat);
377 if (ret)
378 goto err_clr_dma_flag;
379
380 return 0;
381
382 err_clr_dma_flag:
383 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
384 nfc->regs + NFC_REG_CTL);
385
386 err_unmap_buf:
387 dma_unmap_sg(nfc->dev, sg, 1, ddir);
388 return ret;
389 }
390
sunxi_nfc_dma_op_cleanup(struct mtd_info * mtd,enum dma_data_direction ddir,struct scatterlist * sg)391 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
392 enum dma_data_direction ddir,
393 struct scatterlist *sg)
394 {
395 struct nand_chip *nand = mtd_to_nand(mtd);
396 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
397
398 dma_unmap_sg(nfc->dev, sg, 1, ddir);
399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
400 nfc->regs + NFC_REG_CTL);
401 }
402
sunxi_nfc_dev_ready(struct mtd_info * mtd)403 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
404 {
405 struct nand_chip *nand = mtd_to_nand(mtd);
406 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
407 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
408 u32 mask;
409
410 if (sunxi_nand->selected < 0)
411 return 0;
412
413 if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) {
414 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
415 return 0;
416 }
417
418 mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb);
419
420 return !!(readl(nfc->regs + NFC_REG_ST) & mask);
421 }
422
sunxi_nfc_select_chip(struct mtd_info * mtd,int chip)423 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
424 {
425 struct nand_chip *nand = mtd_to_nand(mtd);
426 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
427 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
428 struct sunxi_nand_chip_sel *sel;
429 u32 ctl;
430
431 if (chip > 0 && chip >= sunxi_nand->nsels)
432 return;
433
434 if (chip == sunxi_nand->selected)
435 return;
436
437 ctl = readl(nfc->regs + NFC_REG_CTL) &
438 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
439
440 if (chip >= 0) {
441 sel = &sunxi_nand->sels[chip];
442
443 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
444 NFC_PAGE_SHIFT(nand->page_shift);
445 if (sel->rb < 0) {
446 nand->dev_ready = NULL;
447 } else {
448 nand->dev_ready = sunxi_nfc_dev_ready;
449 ctl |= NFC_RB_SEL(sel->rb);
450 }
451
452 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
453
454 if (nfc->clk_rate != sunxi_nand->clk_rate) {
455 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
456 nfc->clk_rate = sunxi_nand->clk_rate;
457 }
458 }
459
460 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
461 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
462 writel(ctl, nfc->regs + NFC_REG_CTL);
463
464 sunxi_nand->selected = chip;
465 }
466
sunxi_nfc_read_buf(struct mtd_info * mtd,uint8_t * buf,int len)467 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
468 {
469 struct nand_chip *nand = mtd_to_nand(mtd);
470 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
471 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
472 int ret;
473 int cnt;
474 int offs = 0;
475 u32 tmp;
476
477 while (len > offs) {
478 bool poll = false;
479
480 cnt = min(len - offs, NFC_SRAM_SIZE);
481
482 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
483 if (ret)
484 break;
485
486 writel(cnt, nfc->regs + NFC_REG_CNT);
487 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
488 writel(tmp, nfc->regs + NFC_REG_CMD);
489
490 /* Arbitrary limit for polling mode */
491 if (cnt < 64)
492 poll = true;
493
494 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
495 if (ret)
496 break;
497
498 if (buf)
499 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
500 cnt);
501 offs += cnt;
502 }
503 }
504
sunxi_nfc_write_buf(struct mtd_info * mtd,const uint8_t * buf,int len)505 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
506 int len)
507 {
508 struct nand_chip *nand = mtd_to_nand(mtd);
509 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
510 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
511 int ret;
512 int cnt;
513 int offs = 0;
514 u32 tmp;
515
516 while (len > offs) {
517 bool poll = false;
518
519 cnt = min(len - offs, NFC_SRAM_SIZE);
520
521 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
522 if (ret)
523 break;
524
525 writel(cnt, nfc->regs + NFC_REG_CNT);
526 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
527 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
528 NFC_ACCESS_DIR;
529 writel(tmp, nfc->regs + NFC_REG_CMD);
530
531 /* Arbitrary limit for polling mode */
532 if (cnt < 64)
533 poll = true;
534
535 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
536 if (ret)
537 break;
538
539 offs += cnt;
540 }
541 }
542
sunxi_nfc_read_byte(struct mtd_info * mtd)543 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
544 {
545 uint8_t ret = 0;
546
547 sunxi_nfc_read_buf(mtd, &ret, 1);
548
549 return ret;
550 }
551
sunxi_nfc_cmd_ctrl(struct mtd_info * mtd,int dat,unsigned int ctrl)552 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
553 unsigned int ctrl)
554 {
555 struct nand_chip *nand = mtd_to_nand(mtd);
556 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
557 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
558 int ret;
559
560 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
561 !(ctrl & (NAND_CLE | NAND_ALE))) {
562 u32 cmd = 0;
563
564 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
565 return;
566
567 if (sunxi_nand->cmd_cycles--)
568 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
569
570 if (sunxi_nand->cmd_cycles--) {
571 cmd |= NFC_SEND_CMD2;
572 writel(sunxi_nand->cmd[1],
573 nfc->regs + NFC_REG_RCMD_SET);
574 }
575
576 sunxi_nand->cmd_cycles = 0;
577
578 if (sunxi_nand->addr_cycles) {
579 cmd |= NFC_SEND_ADR |
580 NFC_ADR_NUM(sunxi_nand->addr_cycles);
581 writel(sunxi_nand->addr[0],
582 nfc->regs + NFC_REG_ADDR_LOW);
583 }
584
585 if (sunxi_nand->addr_cycles > 4)
586 writel(sunxi_nand->addr[1],
587 nfc->regs + NFC_REG_ADDR_HIGH);
588
589 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
590 if (ret)
591 return;
592
593 writel(cmd, nfc->regs + NFC_REG_CMD);
594 sunxi_nand->addr[0] = 0;
595 sunxi_nand->addr[1] = 0;
596 sunxi_nand->addr_cycles = 0;
597 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
598 }
599
600 if (ctrl & NAND_CLE) {
601 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
602 } else if (ctrl & NAND_ALE) {
603 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
604 dat << ((sunxi_nand->addr_cycles % 4) * 8);
605 sunxi_nand->addr_cycles++;
606 }
607 }
608
609 /* These seed values have been extracted from Allwinner's BSP */
610 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
611 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
612 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
613 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
614 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
615 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
616 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
617 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
618 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
619 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
620 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
621 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
622 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
623 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
624 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
625 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
626 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
627 };
628
629 /*
630 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
631 * have been generated using
632 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
633 * the randomizer engine does internally before de/scrambling OOB data.
634 *
635 * Those tables are statically defined to avoid calculating randomizer state
636 * at runtime.
637 */
638 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
639 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
640 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
641 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
642 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
643 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
644 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
645 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
646 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
647 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
648 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
649 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
650 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
651 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
652 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
653 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
654 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
655 };
656
657 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
658 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
659 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
660 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
661 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
662 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
663 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
664 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
665 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
666 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
667 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
668 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
669 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
670 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
671 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
672 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
673 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
674 };
675
sunxi_nfc_randomizer_step(u16 state,int count)676 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
677 {
678 state &= 0x7fff;
679
680 /*
681 * This loop is just a simple implementation of a Fibonacci LFSR using
682 * the x16 + x15 + 1 polynomial.
683 */
684 while (count--)
685 state = ((state >> 1) |
686 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
687
688 return state;
689 }
690
sunxi_nfc_randomizer_state(struct mtd_info * mtd,int page,bool ecc)691 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
692 {
693 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
694 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
695
696 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
697 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
698
699 if (ecc) {
700 if (mtd->ecc_step_size == 512)
701 seeds = sunxi_nfc_randomizer_ecc512_seeds;
702 else
703 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
704 }
705
706 return seeds[page % mod];
707 }
708
sunxi_nfc_randomizer_config(struct mtd_info * mtd,int page,bool ecc)709 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
710 int page, bool ecc)
711 {
712 struct nand_chip *nand = mtd_to_nand(mtd);
713 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
714 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
715 u16 state;
716
717 if (!(nand->options & NAND_NEED_SCRAMBLING))
718 return;
719
720 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
721 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
722 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
723 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
724 }
725
sunxi_nfc_randomizer_enable(struct mtd_info * mtd)726 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
727 {
728 struct nand_chip *nand = mtd_to_nand(mtd);
729 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
730
731 if (!(nand->options & NAND_NEED_SCRAMBLING))
732 return;
733
734 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
735 nfc->regs + NFC_REG_ECC_CTL);
736 }
737
sunxi_nfc_randomizer_disable(struct mtd_info * mtd)738 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
739 {
740 struct nand_chip *nand = mtd_to_nand(mtd);
741 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
742
743 if (!(nand->options & NAND_NEED_SCRAMBLING))
744 return;
745
746 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
747 nfc->regs + NFC_REG_ECC_CTL);
748 }
749
sunxi_nfc_randomize_bbm(struct mtd_info * mtd,int page,u8 * bbm)750 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
751 {
752 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
753
754 bbm[0] ^= state;
755 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
756 }
757
sunxi_nfc_randomizer_write_buf(struct mtd_info * mtd,const uint8_t * buf,int len,bool ecc,int page)758 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
759 const uint8_t *buf, int len,
760 bool ecc, int page)
761 {
762 sunxi_nfc_randomizer_config(mtd, page, ecc);
763 sunxi_nfc_randomizer_enable(mtd);
764 sunxi_nfc_write_buf(mtd, buf, len);
765 sunxi_nfc_randomizer_disable(mtd);
766 }
767
sunxi_nfc_randomizer_read_buf(struct mtd_info * mtd,uint8_t * buf,int len,bool ecc,int page)768 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
769 int len, bool ecc, int page)
770 {
771 sunxi_nfc_randomizer_config(mtd, page, ecc);
772 sunxi_nfc_randomizer_enable(mtd);
773 sunxi_nfc_read_buf(mtd, buf, len);
774 sunxi_nfc_randomizer_disable(mtd);
775 }
776
sunxi_nfc_hw_ecc_enable(struct mtd_info * mtd)777 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
778 {
779 struct nand_chip *nand = mtd_to_nand(mtd);
780 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
781 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
782 u32 ecc_ctl;
783
784 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
785 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
786 NFC_ECC_BLOCK_SIZE_MSK);
787 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
788 NFC_ECC_PIPELINE;
789
790 if (nand->ecc.size == 512)
791 ecc_ctl |= NFC_ECC_BLOCK_512;
792
793 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
794 }
795
sunxi_nfc_hw_ecc_disable(struct mtd_info * mtd)796 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
797 {
798 struct nand_chip *nand = mtd_to_nand(mtd);
799 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
800
801 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
802 nfc->regs + NFC_REG_ECC_CTL);
803 }
804
sunxi_nfc_user_data_to_buf(u32 user_data,u8 * buf)805 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
806 {
807 buf[0] = user_data;
808 buf[1] = user_data >> 8;
809 buf[2] = user_data >> 16;
810 buf[3] = user_data >> 24;
811 }
812
sunxi_nfc_buf_to_user_data(const u8 * buf)813 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
814 {
815 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
816 }
817
sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info * mtd,u8 * oob,int step,bool bbm,int page)818 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
819 int step, bool bbm, int page)
820 {
821 struct nand_chip *nand = mtd_to_nand(mtd);
822 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
823
824 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
825 oob);
826
827 /* De-randomize the Bad Block Marker. */
828 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
829 sunxi_nfc_randomize_bbm(mtd, page, oob);
830 }
831
sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info * mtd,const u8 * oob,int step,bool bbm,int page)832 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
833 const u8 *oob, int step,
834 bool bbm, int page)
835 {
836 struct nand_chip *nand = mtd_to_nand(mtd);
837 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
838 u8 user_data[4];
839
840 /* Randomize the Bad Block Marker. */
841 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
842 memcpy(user_data, oob, sizeof(user_data));
843 sunxi_nfc_randomize_bbm(mtd, page, user_data);
844 oob = user_data;
845 }
846
847 writel(sunxi_nfc_buf_to_user_data(oob),
848 nfc->regs + NFC_REG_USER_DATA(step));
849 }
850
sunxi_nfc_hw_ecc_update_stats(struct mtd_info * mtd,unsigned int * max_bitflips,int ret)851 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
852 unsigned int *max_bitflips, int ret)
853 {
854 if (ret < 0) {
855 mtd->ecc_stats.failed++;
856 } else {
857 mtd->ecc_stats.corrected += ret;
858 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
859 }
860 }
861
sunxi_nfc_hw_ecc_correct(struct mtd_info * mtd,u8 * data,u8 * oob,int step,u32 status,bool * erased)862 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
863 int step, u32 status, bool *erased)
864 {
865 struct nand_chip *nand = mtd_to_nand(mtd);
866 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
867 struct nand_ecc_ctrl *ecc = &nand->ecc;
868 u32 tmp;
869
870 *erased = false;
871
872 if (status & NFC_ECC_ERR(step))
873 return -EBADMSG;
874
875 if (status & NFC_ECC_PAT_FOUND(step)) {
876 u8 pattern;
877
878 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
879 pattern = 0x0;
880 } else {
881 pattern = 0xff;
882 *erased = true;
883 }
884
885 if (data)
886 memset(data, pattern, ecc->size);
887
888 if (oob)
889 memset(oob, pattern, ecc->bytes + 4);
890
891 return 0;
892 }
893
894 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
895
896 return NFC_ECC_ERR_CNT(step, tmp);
897 }
898
sunxi_nfc_hw_ecc_read_chunk(struct mtd_info * mtd,u8 * data,int data_off,u8 * oob,int oob_off,int * cur_off,unsigned int * max_bitflips,bool bbm,bool oob_required,int page)899 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
900 u8 *data, int data_off,
901 u8 *oob, int oob_off,
902 int *cur_off,
903 unsigned int *max_bitflips,
904 bool bbm, bool oob_required, int page)
905 {
906 struct nand_chip *nand = mtd_to_nand(mtd);
907 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
908 struct nand_ecc_ctrl *ecc = &nand->ecc;
909 int raw_mode = 0;
910 bool erased;
911 int ret;
912
913 if (*cur_off != data_off)
914 nand_change_read_column_op(nand, data_off, NULL, 0, false);
915
916 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
917
918 if (data_off + ecc->size != oob_off)
919 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
920
921 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
922 if (ret)
923 return ret;
924
925 sunxi_nfc_randomizer_enable(mtd);
926 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
927 nfc->regs + NFC_REG_CMD);
928
929 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
930 sunxi_nfc_randomizer_disable(mtd);
931 if (ret)
932 return ret;
933
934 *cur_off = oob_off + ecc->bytes + 4;
935
936 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
937 readl(nfc->regs + NFC_REG_ECC_ST),
938 &erased);
939 if (erased)
940 return 1;
941
942 if (ret < 0) {
943 /*
944 * Re-read the data with the randomizer disabled to identify
945 * bitflips in erased pages.
946 */
947 if (nand->options & NAND_NEED_SCRAMBLING)
948 nand_change_read_column_op(nand, data_off, data,
949 ecc->size, false);
950 else
951 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
952 ecc->size);
953
954 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
955 false);
956
957 ret = nand_check_erased_ecc_chunk(data, ecc->size,
958 oob, ecc->bytes + 4,
959 NULL, 0, ecc->strength);
960 if (ret >= 0)
961 raw_mode = 1;
962 } else {
963 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
964
965 if (oob_required) {
966 nand_change_read_column_op(nand, oob_off, NULL, 0,
967 false);
968 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
969 true, page);
970
971 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
972 bbm, page);
973 }
974 }
975
976 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
977
978 return raw_mode;
979 }
980
sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info * mtd,u8 * oob,int * cur_off,bool randomize,int page)981 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
982 u8 *oob, int *cur_off,
983 bool randomize, int page)
984 {
985 struct nand_chip *nand = mtd_to_nand(mtd);
986 struct nand_ecc_ctrl *ecc = &nand->ecc;
987 int offset = ((ecc->bytes + 4) * ecc->steps);
988 int len = mtd->oobsize - offset;
989
990 if (len <= 0)
991 return;
992
993 if (!cur_off || *cur_off != offset)
994 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
995 false);
996
997 if (!randomize)
998 sunxi_nfc_read_buf(mtd, oob + offset, len);
999 else
1000 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1001 false, page);
1002
1003 if (cur_off)
1004 *cur_off = mtd->oobsize + mtd->writesize;
1005 }
1006
sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info * mtd,uint8_t * buf,int oob_required,int page,int nchunks)1007 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1008 int oob_required, int page,
1009 int nchunks)
1010 {
1011 struct nand_chip *nand = mtd_to_nand(mtd);
1012 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1013 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1014 struct nand_ecc_ctrl *ecc = &nand->ecc;
1015 unsigned int max_bitflips = 0;
1016 int ret, i, raw_mode = 0;
1017 struct scatterlist sg;
1018 u32 status;
1019
1020 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1021 if (ret)
1022 return ret;
1023
1024 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1025 DMA_FROM_DEVICE, &sg);
1026 if (ret)
1027 return ret;
1028
1029 sunxi_nfc_hw_ecc_enable(mtd);
1030 sunxi_nfc_randomizer_config(mtd, page, false);
1031 sunxi_nfc_randomizer_enable(mtd);
1032
1033 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1034 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1035
1036 dma_async_issue_pending(nfc->dmac);
1037
1038 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1039 nfc->regs + NFC_REG_CMD);
1040
1041 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1042 if (ret)
1043 dmaengine_terminate_all(nfc->dmac);
1044
1045 sunxi_nfc_randomizer_disable(mtd);
1046 sunxi_nfc_hw_ecc_disable(mtd);
1047
1048 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1049
1050 if (ret)
1051 return ret;
1052
1053 status = readl(nfc->regs + NFC_REG_ECC_ST);
1054
1055 for (i = 0; i < nchunks; i++) {
1056 int data_off = i * ecc->size;
1057 int oob_off = i * (ecc->bytes + 4);
1058 u8 *data = buf + data_off;
1059 u8 *oob = nand->oob_poi + oob_off;
1060 bool erased;
1061
1062 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1063 oob_required ? oob : NULL,
1064 i, status, &erased);
1065
1066 /* ECC errors are handled in the second loop. */
1067 if (ret < 0)
1068 continue;
1069
1070 if (oob_required && !erased) {
1071 /* TODO: use DMA to retrieve OOB */
1072 nand_change_read_column_op(nand,
1073 mtd->writesize + oob_off,
1074 oob, ecc->bytes + 4, false);
1075
1076 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1077 !i, page);
1078 }
1079
1080 if (erased)
1081 raw_mode = 1;
1082
1083 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1084 }
1085
1086 if (status & NFC_ECC_ERR_MSK) {
1087 for (i = 0; i < nchunks; i++) {
1088 int data_off = i * ecc->size;
1089 int oob_off = i * (ecc->bytes + 4);
1090 u8 *data = buf + data_off;
1091 u8 *oob = nand->oob_poi + oob_off;
1092
1093 if (!(status & NFC_ECC_ERR(i)))
1094 continue;
1095
1096 /*
1097 * Re-read the data with the randomizer disabled to
1098 * identify bitflips in erased pages.
1099 * TODO: use DMA to read page in raw mode
1100 */
1101 if (randomized)
1102 nand_change_read_column_op(nand, data_off,
1103 data, ecc->size,
1104 false);
1105
1106 /* TODO: use DMA to retrieve OOB */
1107 nand_change_read_column_op(nand,
1108 mtd->writesize + oob_off,
1109 oob, ecc->bytes + 4, false);
1110
1111 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1112 oob, ecc->bytes + 4,
1113 NULL, 0,
1114 ecc->strength);
1115 if (ret >= 0)
1116 raw_mode = 1;
1117
1118 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1119 }
1120 }
1121
1122 if (oob_required)
1123 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1124 NULL, !raw_mode,
1125 page);
1126
1127 return max_bitflips;
1128 }
1129
sunxi_nfc_hw_ecc_write_chunk(struct mtd_info * mtd,const u8 * data,int data_off,const u8 * oob,int oob_off,int * cur_off,bool bbm,int page)1130 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1131 const u8 *data, int data_off,
1132 const u8 *oob, int oob_off,
1133 int *cur_off, bool bbm,
1134 int page)
1135 {
1136 struct nand_chip *nand = mtd_to_nand(mtd);
1137 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1138 struct nand_ecc_ctrl *ecc = &nand->ecc;
1139 int ret;
1140
1141 if (data_off != *cur_off)
1142 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1143
1144 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1145
1146 if (data_off + ecc->size != oob_off)
1147 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1148
1149 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1150 if (ret)
1151 return ret;
1152
1153 sunxi_nfc_randomizer_enable(mtd);
1154 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1155
1156 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1157 NFC_ACCESS_DIR | NFC_ECC_OP,
1158 nfc->regs + NFC_REG_CMD);
1159
1160 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1161 sunxi_nfc_randomizer_disable(mtd);
1162 if (ret)
1163 return ret;
1164
1165 *cur_off = oob_off + ecc->bytes + 4;
1166
1167 return 0;
1168 }
1169
sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info * mtd,u8 * oob,int * cur_off,int page)1170 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1171 u8 *oob, int *cur_off,
1172 int page)
1173 {
1174 struct nand_chip *nand = mtd_to_nand(mtd);
1175 struct nand_ecc_ctrl *ecc = &nand->ecc;
1176 int offset = ((ecc->bytes + 4) * ecc->steps);
1177 int len = mtd->oobsize - offset;
1178
1179 if (len <= 0)
1180 return;
1181
1182 if (!cur_off || *cur_off != offset)
1183 nand_change_write_column_op(nand, offset + mtd->writesize,
1184 NULL, 0, false);
1185
1186 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1187
1188 if (cur_off)
1189 *cur_off = mtd->oobsize + mtd->writesize;
1190 }
1191
sunxi_nfc_hw_ecc_read_page(struct mtd_info * mtd,struct nand_chip * chip,uint8_t * buf,int oob_required,int page)1192 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1193 struct nand_chip *chip, uint8_t *buf,
1194 int oob_required, int page)
1195 {
1196 struct nand_ecc_ctrl *ecc = &chip->ecc;
1197 unsigned int max_bitflips = 0;
1198 int ret, i, cur_off = 0;
1199 bool raw_mode = false;
1200
1201 nand_read_page_op(chip, page, 0, NULL, 0);
1202
1203 sunxi_nfc_hw_ecc_enable(mtd);
1204
1205 for (i = 0; i < ecc->steps; i++) {
1206 int data_off = i * ecc->size;
1207 int oob_off = i * (ecc->bytes + 4);
1208 u8 *data = buf + data_off;
1209 u8 *oob = chip->oob_poi + oob_off;
1210
1211 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1212 oob_off + mtd->writesize,
1213 &cur_off, &max_bitflips,
1214 !i, oob_required, page);
1215 if (ret < 0)
1216 return ret;
1217 else if (ret)
1218 raw_mode = true;
1219 }
1220
1221 if (oob_required)
1222 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1223 !raw_mode, page);
1224
1225 sunxi_nfc_hw_ecc_disable(mtd);
1226
1227 return max_bitflips;
1228 }
1229
sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info * mtd,struct nand_chip * chip,u8 * buf,int oob_required,int page)1230 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1231 struct nand_chip *chip, u8 *buf,
1232 int oob_required, int page)
1233 {
1234 int ret;
1235
1236 nand_read_page_op(chip, page, 0, NULL, 0);
1237
1238 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1239 chip->ecc.steps);
1240 if (ret >= 0)
1241 return ret;
1242
1243 /* Fallback to PIO mode */
1244 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1245 }
1246
sunxi_nfc_hw_ecc_read_subpage(struct mtd_info * mtd,struct nand_chip * chip,u32 data_offs,u32 readlen,u8 * bufpoi,int page)1247 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1248 struct nand_chip *chip,
1249 u32 data_offs, u32 readlen,
1250 u8 *bufpoi, int page)
1251 {
1252 struct nand_ecc_ctrl *ecc = &chip->ecc;
1253 int ret, i, cur_off = 0;
1254 unsigned int max_bitflips = 0;
1255
1256 nand_read_page_op(chip, page, 0, NULL, 0);
1257
1258 sunxi_nfc_hw_ecc_enable(mtd);
1259
1260 for (i = data_offs / ecc->size;
1261 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1262 int data_off = i * ecc->size;
1263 int oob_off = i * (ecc->bytes + 4);
1264 u8 *data = bufpoi + data_off;
1265 u8 *oob = chip->oob_poi + oob_off;
1266
1267 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1268 oob,
1269 oob_off + mtd->writesize,
1270 &cur_off, &max_bitflips, !i,
1271 false, page);
1272 if (ret < 0)
1273 return ret;
1274 }
1275
1276 sunxi_nfc_hw_ecc_disable(mtd);
1277
1278 return max_bitflips;
1279 }
1280
sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info * mtd,struct nand_chip * chip,u32 data_offs,u32 readlen,u8 * buf,int page)1281 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1282 struct nand_chip *chip,
1283 u32 data_offs, u32 readlen,
1284 u8 *buf, int page)
1285 {
1286 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1287 int ret;
1288
1289 nand_read_page_op(chip, page, 0, NULL, 0);
1290
1291 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1292 if (ret >= 0)
1293 return ret;
1294
1295 /* Fallback to PIO mode */
1296 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1297 buf, page);
1298 }
1299
sunxi_nfc_hw_ecc_write_page(struct mtd_info * mtd,struct nand_chip * chip,const uint8_t * buf,int oob_required,int page)1300 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1301 struct nand_chip *chip,
1302 const uint8_t *buf, int oob_required,
1303 int page)
1304 {
1305 struct nand_ecc_ctrl *ecc = &chip->ecc;
1306 int ret, i, cur_off = 0;
1307
1308 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1309
1310 sunxi_nfc_hw_ecc_enable(mtd);
1311
1312 for (i = 0; i < ecc->steps; i++) {
1313 int data_off = i * ecc->size;
1314 int oob_off = i * (ecc->bytes + 4);
1315 const u8 *data = buf + data_off;
1316 const u8 *oob = chip->oob_poi + oob_off;
1317
1318 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1319 oob_off + mtd->writesize,
1320 &cur_off, !i, page);
1321 if (ret)
1322 return ret;
1323 }
1324
1325 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1326 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1327 &cur_off, page);
1328
1329 sunxi_nfc_hw_ecc_disable(mtd);
1330
1331 return nand_prog_page_end_op(chip);
1332 }
1333
sunxi_nfc_hw_ecc_write_subpage(struct mtd_info * mtd,struct nand_chip * chip,u32 data_offs,u32 data_len,const u8 * buf,int oob_required,int page)1334 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1335 struct nand_chip *chip,
1336 u32 data_offs, u32 data_len,
1337 const u8 *buf, int oob_required,
1338 int page)
1339 {
1340 struct nand_ecc_ctrl *ecc = &chip->ecc;
1341 int ret, i, cur_off = 0;
1342
1343 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1344
1345 sunxi_nfc_hw_ecc_enable(mtd);
1346
1347 for (i = data_offs / ecc->size;
1348 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1349 int data_off = i * ecc->size;
1350 int oob_off = i * (ecc->bytes + 4);
1351 const u8 *data = buf + data_off;
1352 const u8 *oob = chip->oob_poi + oob_off;
1353
1354 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1355 oob_off + mtd->writesize,
1356 &cur_off, !i, page);
1357 if (ret)
1358 return ret;
1359 }
1360
1361 sunxi_nfc_hw_ecc_disable(mtd);
1362
1363 return nand_prog_page_end_op(chip);
1364 }
1365
sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info * mtd,struct nand_chip * chip,const u8 * buf,int oob_required,int page)1366 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1367 struct nand_chip *chip,
1368 const u8 *buf,
1369 int oob_required,
1370 int page)
1371 {
1372 struct nand_chip *nand = mtd_to_nand(mtd);
1373 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1374 struct nand_ecc_ctrl *ecc = &nand->ecc;
1375 struct scatterlist sg;
1376 int ret, i;
1377
1378 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1379 if (ret)
1380 return ret;
1381
1382 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1383 DMA_TO_DEVICE, &sg);
1384 if (ret)
1385 goto pio_fallback;
1386
1387 for (i = 0; i < ecc->steps; i++) {
1388 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1389
1390 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1391 }
1392
1393 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1394
1395 sunxi_nfc_hw_ecc_enable(mtd);
1396 sunxi_nfc_randomizer_config(mtd, page, false);
1397 sunxi_nfc_randomizer_enable(mtd);
1398
1399 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1400 nfc->regs + NFC_REG_RCMD_SET);
1401
1402 dma_async_issue_pending(nfc->dmac);
1403
1404 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1405 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1406 nfc->regs + NFC_REG_CMD);
1407
1408 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1409 if (ret)
1410 dmaengine_terminate_all(nfc->dmac);
1411
1412 sunxi_nfc_randomizer_disable(mtd);
1413 sunxi_nfc_hw_ecc_disable(mtd);
1414
1415 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1416
1417 if (ret)
1418 return ret;
1419
1420 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1421 /* TODO: use DMA to transfer extra OOB bytes ? */
1422 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1423 NULL, page);
1424
1425 return nand_prog_page_end_op(chip);
1426
1427 pio_fallback:
1428 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1429 }
1430
sunxi_nfc_hw_ecc_read_oob(struct mtd_info * mtd,struct nand_chip * chip,int page)1431 static int sunxi_nfc_hw_ecc_read_oob(struct mtd_info *mtd,
1432 struct nand_chip *chip,
1433 int page)
1434 {
1435 chip->pagebuf = -1;
1436
1437 return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page);
1438 }
1439
sunxi_nfc_hw_ecc_write_oob(struct mtd_info * mtd,struct nand_chip * chip,int page)1440 static int sunxi_nfc_hw_ecc_write_oob(struct mtd_info *mtd,
1441 struct nand_chip *chip,
1442 int page)
1443 {
1444 int ret;
1445
1446 chip->pagebuf = -1;
1447
1448 memset(chip->data_buf, 0xff, mtd->writesize);
1449 ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page);
1450 if (ret)
1451 return ret;
1452
1453 /* Send command to program the OOB data */
1454 return nand_prog_page_end_op(chip);
1455 }
1456
1457 static const s32 tWB_lut[] = {6, 12, 16, 20};
1458 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1459
_sunxi_nand_lookup_timing(const s32 * lut,int lut_size,u32 duration,u32 clk_period)1460 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1461 u32 clk_period)
1462 {
1463 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1464 int i;
1465
1466 for (i = 0; i < lut_size; i++) {
1467 if (clk_cycles <= lut[i])
1468 return i;
1469 }
1470
1471 /* Doesn't fit */
1472 return -EINVAL;
1473 }
1474
1475 #define sunxi_nand_lookup_timing(l, p, c) \
1476 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1477
sunxi_nfc_setup_data_interface(struct mtd_info * mtd,int csline,const struct nand_data_interface * conf)1478 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline,
1479 const struct nand_data_interface *conf)
1480 {
1481 struct nand_chip *nand = mtd_to_nand(mtd);
1482 struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1483 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1484 const struct nand_sdr_timings *timings;
1485 u32 min_clk_period = 0;
1486 s32 tWB, tADL, tWHR, tRHW, tCAD;
1487 long real_clk_rate;
1488
1489 timings = nand_get_sdr_timings(conf);
1490 if (IS_ERR(timings))
1491 return -ENOTSUPP;
1492
1493 /* T1 <=> tCLS */
1494 if (timings->tCLS_min > min_clk_period)
1495 min_clk_period = timings->tCLS_min;
1496
1497 /* T2 <=> tCLH */
1498 if (timings->tCLH_min > min_clk_period)
1499 min_clk_period = timings->tCLH_min;
1500
1501 /* T3 <=> tCS */
1502 if (timings->tCS_min > min_clk_period)
1503 min_clk_period = timings->tCS_min;
1504
1505 /* T4 <=> tCH */
1506 if (timings->tCH_min > min_clk_period)
1507 min_clk_period = timings->tCH_min;
1508
1509 /* T5 <=> tWP */
1510 if (timings->tWP_min > min_clk_period)
1511 min_clk_period = timings->tWP_min;
1512
1513 /* T6 <=> tWH */
1514 if (timings->tWH_min > min_clk_period)
1515 min_clk_period = timings->tWH_min;
1516
1517 /* T7 <=> tALS */
1518 if (timings->tALS_min > min_clk_period)
1519 min_clk_period = timings->tALS_min;
1520
1521 /* T8 <=> tDS */
1522 if (timings->tDS_min > min_clk_period)
1523 min_clk_period = timings->tDS_min;
1524
1525 /* T9 <=> tDH */
1526 if (timings->tDH_min > min_clk_period)
1527 min_clk_period = timings->tDH_min;
1528
1529 /* T10 <=> tRR */
1530 if (timings->tRR_min > (min_clk_period * 3))
1531 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1532
1533 /* T11 <=> tALH */
1534 if (timings->tALH_min > min_clk_period)
1535 min_clk_period = timings->tALH_min;
1536
1537 /* T12 <=> tRP */
1538 if (timings->tRP_min > min_clk_period)
1539 min_clk_period = timings->tRP_min;
1540
1541 /* T13 <=> tREH */
1542 if (timings->tREH_min > min_clk_period)
1543 min_clk_period = timings->tREH_min;
1544
1545 /* T14 <=> tRC */
1546 if (timings->tRC_min > (min_clk_period * 2))
1547 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1548
1549 /* T15 <=> tWC */
1550 if (timings->tWC_min > (min_clk_period * 2))
1551 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1552
1553 /* T16 - T19 + tCAD */
1554 if (timings->tWB_max > (min_clk_period * 20))
1555 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1556
1557 if (timings->tADL_min > (min_clk_period * 32))
1558 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1559
1560 if (timings->tWHR_min > (min_clk_period * 32))
1561 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1562
1563 if (timings->tRHW_min > (min_clk_period * 20))
1564 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1565
1566 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1567 min_clk_period);
1568 if (tWB < 0) {
1569 dev_err(nfc->dev, "unsupported tWB\n");
1570 return tWB;
1571 }
1572
1573 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1574 if (tADL > 3) {
1575 dev_err(nfc->dev, "unsupported tADL\n");
1576 return -EINVAL;
1577 }
1578
1579 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1580 if (tWHR > 3) {
1581 dev_err(nfc->dev, "unsupported tWHR\n");
1582 return -EINVAL;
1583 }
1584
1585 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1586 min_clk_period);
1587 if (tRHW < 0) {
1588 dev_err(nfc->dev, "unsupported tRHW\n");
1589 return tRHW;
1590 }
1591
1592 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1593 return 0;
1594
1595 /*
1596 * TODO: according to ONFI specs this value only applies for DDR NAND,
1597 * but Allwinner seems to set this to 0x7. Mimic them for now.
1598 */
1599 tCAD = 0x7;
1600
1601 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1602 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1603
1604 /* Convert min_clk_period from picoseconds to nanoseconds */
1605 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1606
1607 /*
1608 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1609 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1610 * This new formula was verified with a scope and validated by
1611 * Allwinner engineers.
1612 */
1613 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1614 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1615 if (real_clk_rate <= 0) {
1616 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate);
1617 return -EINVAL;
1618 }
1619
1620 /*
1621 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1622 * output cycle timings shall be used if the host drives tRC less than
1623 * 30 ns.
1624 */
1625 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1626 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1627 NFC_TIMING_CTL_EDO : 0;
1628
1629 return 0;
1630 }
1631
sunxi_nand_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1632 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1633 struct mtd_oob_region *oobregion)
1634 {
1635 struct nand_chip *nand = mtd_to_nand(mtd);
1636 struct nand_ecc_ctrl *ecc = &nand->ecc;
1637
1638 if (section >= ecc->steps)
1639 return -ERANGE;
1640
1641 oobregion->offset = section * (ecc->bytes + 4) + 4;
1642 oobregion->length = ecc->bytes;
1643
1644 return 0;
1645 }
1646
sunxi_nand_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)1647 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1648 struct mtd_oob_region *oobregion)
1649 {
1650 struct nand_chip *nand = mtd_to_nand(mtd);
1651 struct nand_ecc_ctrl *ecc = &nand->ecc;
1652
1653 if (section > ecc->steps)
1654 return -ERANGE;
1655
1656 /*
1657 * The first 2 bytes are used for BB markers, hence we
1658 * only have 2 bytes available in the first user data
1659 * section.
1660 */
1661 if (!section && ecc->mode == NAND_ECC_HW) {
1662 oobregion->offset = 2;
1663 oobregion->length = 2;
1664
1665 return 0;
1666 }
1667
1668 oobregion->offset = section * (ecc->bytes + 4);
1669
1670 if (section < ecc->steps)
1671 oobregion->length = 4;
1672 else
1673 oobregion->offset = mtd->oobsize - oobregion->offset;
1674
1675 return 0;
1676 }
1677
1678 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1679 .ecc = sunxi_nand_ooblayout_ecc,
1680 .free = sunxi_nand_ooblayout_free,
1681 };
1682
sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl * ecc)1683 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1684 {
1685 kfree(ecc->priv);
1686 }
1687
sunxi_nand_hw_ecc_ctrl_init(struct mtd_info * mtd,struct nand_ecc_ctrl * ecc,struct device_node * np)1688 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1689 struct nand_ecc_ctrl *ecc,
1690 struct device_node *np)
1691 {
1692 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1693 struct nand_chip *nand = mtd_to_nand(mtd);
1694 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1695 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1696 struct sunxi_nand_hw_ecc *data;
1697 int nsectors;
1698 int ret;
1699 int i;
1700
1701 if (ecc->options & NAND_ECC_MAXIMIZE) {
1702 int bytes;
1703
1704 ecc->size = 1024;
1705 nsectors = mtd->writesize / ecc->size;
1706
1707 /* Reserve 2 bytes for the BBM */
1708 bytes = (mtd->oobsize - 2) / nsectors;
1709
1710 /* 4 non-ECC bytes are added before each ECC bytes section */
1711 bytes -= 4;
1712
1713 /* and bytes has to be even. */
1714 if (bytes % 2)
1715 bytes--;
1716
1717 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1718
1719 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1720 if (strengths[i] > ecc->strength)
1721 break;
1722 }
1723
1724 if (!i)
1725 ecc->strength = 0;
1726 else
1727 ecc->strength = strengths[i - 1];
1728 }
1729
1730 if (ecc->size != 512 && ecc->size != 1024)
1731 return -EINVAL;
1732
1733 data = kzalloc(sizeof(*data), GFP_KERNEL);
1734 if (!data)
1735 return -ENOMEM;
1736
1737 /* Prefer 1k ECC chunk over 512 ones */
1738 if (ecc->size == 512 && mtd->writesize > 512) {
1739 ecc->size = 1024;
1740 ecc->strength *= 2;
1741 }
1742
1743 /* Add ECC info retrieval from DT */
1744 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1745 if (ecc->strength <= strengths[i]) {
1746 /*
1747 * Update ecc->strength value with the actual strength
1748 * that will be used by the ECC engine.
1749 */
1750 ecc->strength = strengths[i];
1751 break;
1752 }
1753 }
1754
1755 if (i >= ARRAY_SIZE(strengths)) {
1756 dev_err(nfc->dev, "unsupported strength\n");
1757 ret = -ENOTSUPP;
1758 goto err;
1759 }
1760
1761 data->mode = i;
1762
1763 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1764 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1765
1766 /* HW ECC always work with even numbers of ECC bytes */
1767 ecc->bytes = ALIGN(ecc->bytes, 2);
1768
1769 nsectors = mtd->writesize / ecc->size;
1770
1771 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1772 ret = -EINVAL;
1773 goto err;
1774 }
1775
1776 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1777 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1778 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1779 ecc->priv = data;
1780
1781 if (nfc->dmac) {
1782 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1783 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1784 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1785 nand->options |= NAND_USE_BOUNCE_BUFFER;
1786 } else {
1787 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1788 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1789 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1790 }
1791
1792 /* TODO: support DMA for raw accesses and subpage write */
1793 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1794 ecc->read_oob_raw = nand_read_oob_std;
1795 ecc->write_oob_raw = nand_write_oob_std;
1796
1797 return 0;
1798
1799 err:
1800 kfree(data);
1801
1802 return ret;
1803 }
1804
sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl * ecc)1805 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1806 {
1807 switch (ecc->mode) {
1808 case NAND_ECC_HW:
1809 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1810 break;
1811 case NAND_ECC_NONE:
1812 default:
1813 break;
1814 }
1815 }
1816
sunxi_nand_attach_chip(struct nand_chip * nand)1817 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1818 {
1819 struct mtd_info *mtd = nand_to_mtd(nand);
1820 struct nand_ecc_ctrl *ecc = &nand->ecc;
1821 struct device_node *np = nand_get_flash_node(nand);
1822 int ret;
1823
1824 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1825 nand->bbt_options |= NAND_BBT_NO_OOB;
1826
1827 if (nand->options & NAND_NEED_SCRAMBLING)
1828 nand->options |= NAND_NO_SUBPAGE_WRITE;
1829
1830 nand->options |= NAND_SUBPAGE_READ;
1831
1832 if (!ecc->size) {
1833 ecc->size = nand->ecc_step_ds;
1834 ecc->strength = nand->ecc_strength_ds;
1835 }
1836
1837 if (!ecc->size || !ecc->strength)
1838 return -EINVAL;
1839
1840 switch (ecc->mode) {
1841 case NAND_ECC_HW:
1842 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1843 if (ret)
1844 return ret;
1845 break;
1846 case NAND_ECC_NONE:
1847 case NAND_ECC_SOFT:
1848 break;
1849 default:
1850 return -EINVAL;
1851 }
1852
1853 return 0;
1854 }
1855
1856 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1857 .attach_chip = sunxi_nand_attach_chip,
1858 };
1859
sunxi_nand_chip_init(struct device * dev,struct sunxi_nfc * nfc,struct device_node * np)1860 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1861 struct device_node *np)
1862 {
1863 struct sunxi_nand_chip *chip;
1864 struct mtd_info *mtd;
1865 struct nand_chip *nand;
1866 int nsels;
1867 int ret;
1868 int i;
1869 u32 tmp;
1870
1871 if (!of_get_property(np, "reg", &nsels))
1872 return -EINVAL;
1873
1874 nsels /= sizeof(u32);
1875 if (!nsels) {
1876 dev_err(dev, "invalid reg property size\n");
1877 return -EINVAL;
1878 }
1879
1880 chip = devm_kzalloc(dev,
1881 sizeof(*chip) +
1882 (nsels * sizeof(struct sunxi_nand_chip_sel)),
1883 GFP_KERNEL);
1884 if (!chip) {
1885 dev_err(dev, "could not allocate chip\n");
1886 return -ENOMEM;
1887 }
1888
1889 chip->nsels = nsels;
1890 chip->selected = -1;
1891
1892 for (i = 0; i < nsels; i++) {
1893 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1894 if (ret) {
1895 dev_err(dev, "could not retrieve reg property: %d\n",
1896 ret);
1897 return ret;
1898 }
1899
1900 if (tmp > NFC_MAX_CS) {
1901 dev_err(dev,
1902 "invalid reg value: %u (max CS = 7)\n",
1903 tmp);
1904 return -EINVAL;
1905 }
1906
1907 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1908 dev_err(dev, "CS %d already assigned\n", tmp);
1909 return -EINVAL;
1910 }
1911
1912 chip->sels[i].cs = tmp;
1913
1914 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1915 tmp < 2)
1916 chip->sels[i].rb = tmp;
1917 else
1918 chip->sels[i].rb = -1;
1919 }
1920
1921 nand = &chip->nand;
1922 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1923 nand->chip_delay = 200;
1924 nand->controller = &nfc->controller;
1925 nand->controller->ops = &sunxi_nand_controller_ops;
1926
1927 /*
1928 * Set the ECC mode to the default value in case nothing is specified
1929 * in the DT.
1930 */
1931 nand->ecc.mode = NAND_ECC_HW;
1932 nand_set_flash_node(nand, np);
1933 nand->select_chip = sunxi_nfc_select_chip;
1934 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
1935 nand->read_buf = sunxi_nfc_read_buf;
1936 nand->write_buf = sunxi_nfc_write_buf;
1937 nand->read_byte = sunxi_nfc_read_byte;
1938 nand->setup_data_interface = sunxi_nfc_setup_data_interface;
1939
1940 mtd = nand_to_mtd(nand);
1941 mtd->dev.parent = dev;
1942
1943 ret = nand_scan(mtd, nsels);
1944 if (ret)
1945 return ret;
1946
1947 ret = mtd_device_register(mtd, NULL, 0);
1948 if (ret) {
1949 dev_err(dev, "failed to register mtd device: %d\n", ret);
1950 nand_release(mtd);
1951 return ret;
1952 }
1953
1954 list_add_tail(&chip->node, &nfc->chips);
1955
1956 return 0;
1957 }
1958
sunxi_nand_chips_init(struct device * dev,struct sunxi_nfc * nfc)1959 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
1960 {
1961 struct device_node *np = dev->of_node;
1962 struct device_node *nand_np;
1963 int nchips = of_get_child_count(np);
1964 int ret;
1965
1966 if (nchips > 8) {
1967 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
1968 return -EINVAL;
1969 }
1970
1971 for_each_child_of_node(np, nand_np) {
1972 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
1973 if (ret) {
1974 of_node_put(nand_np);
1975 return ret;
1976 }
1977 }
1978
1979 return 0;
1980 }
1981
sunxi_nand_chips_cleanup(struct sunxi_nfc * nfc)1982 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1983 {
1984 struct sunxi_nand_chip *chip;
1985
1986 while (!list_empty(&nfc->chips)) {
1987 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
1988 node);
1989 nand_release(nand_to_mtd(&chip->nand));
1990 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
1991 list_del(&chip->node);
1992 }
1993 }
1994
sunxi_nfc_probe(struct platform_device * pdev)1995 static int sunxi_nfc_probe(struct platform_device *pdev)
1996 {
1997 struct device *dev = &pdev->dev;
1998 struct resource *r;
1999 struct sunxi_nfc *nfc;
2000 int irq;
2001 int ret;
2002
2003 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2004 if (!nfc)
2005 return -ENOMEM;
2006
2007 nfc->dev = dev;
2008 nand_controller_init(&nfc->controller);
2009 INIT_LIST_HEAD(&nfc->chips);
2010
2011 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2012 nfc->regs = devm_ioremap_resource(dev, r);
2013 if (IS_ERR(nfc->regs))
2014 return PTR_ERR(nfc->regs);
2015
2016 irq = platform_get_irq(pdev, 0);
2017 if (irq < 0) {
2018 dev_err(dev, "failed to retrieve irq\n");
2019 return irq;
2020 }
2021
2022 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2023 if (IS_ERR(nfc->ahb_clk)) {
2024 dev_err(dev, "failed to retrieve ahb clk\n");
2025 return PTR_ERR(nfc->ahb_clk);
2026 }
2027
2028 ret = clk_prepare_enable(nfc->ahb_clk);
2029 if (ret)
2030 return ret;
2031
2032 nfc->mod_clk = devm_clk_get(dev, "mod");
2033 if (IS_ERR(nfc->mod_clk)) {
2034 dev_err(dev, "failed to retrieve mod clk\n");
2035 ret = PTR_ERR(nfc->mod_clk);
2036 goto out_ahb_clk_unprepare;
2037 }
2038
2039 ret = clk_prepare_enable(nfc->mod_clk);
2040 if (ret)
2041 goto out_ahb_clk_unprepare;
2042
2043 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2044 if (IS_ERR(nfc->reset)) {
2045 ret = PTR_ERR(nfc->reset);
2046 goto out_mod_clk_unprepare;
2047 }
2048
2049 ret = reset_control_deassert(nfc->reset);
2050 if (ret) {
2051 dev_err(dev, "reset err %d\n", ret);
2052 goto out_mod_clk_unprepare;
2053 }
2054
2055 ret = sunxi_nfc_rst(nfc);
2056 if (ret)
2057 goto out_ahb_reset_reassert;
2058
2059 writel(0, nfc->regs + NFC_REG_INT);
2060 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2061 0, "sunxi-nand", nfc);
2062 if (ret)
2063 goto out_ahb_reset_reassert;
2064
2065 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2066 if (nfc->dmac) {
2067 struct dma_slave_config dmac_cfg = { };
2068
2069 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2070 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2071 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2072 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2073 dmac_cfg.src_maxburst = 4;
2074 dmac_cfg.dst_maxburst = 4;
2075 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2076 } else {
2077 dev_warn(dev, "failed to request rxtx DMA channel\n");
2078 }
2079
2080 platform_set_drvdata(pdev, nfc);
2081
2082 ret = sunxi_nand_chips_init(dev, nfc);
2083 if (ret) {
2084 dev_err(dev, "failed to init nand chips\n");
2085 goto out_release_dmac;
2086 }
2087
2088 return 0;
2089
2090 out_release_dmac:
2091 if (nfc->dmac)
2092 dma_release_channel(nfc->dmac);
2093 out_ahb_reset_reassert:
2094 reset_control_assert(nfc->reset);
2095 out_mod_clk_unprepare:
2096 clk_disable_unprepare(nfc->mod_clk);
2097 out_ahb_clk_unprepare:
2098 clk_disable_unprepare(nfc->ahb_clk);
2099
2100 return ret;
2101 }
2102
sunxi_nfc_remove(struct platform_device * pdev)2103 static int sunxi_nfc_remove(struct platform_device *pdev)
2104 {
2105 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2106
2107 sunxi_nand_chips_cleanup(nfc);
2108
2109 reset_control_assert(nfc->reset);
2110
2111 if (nfc->dmac)
2112 dma_release_channel(nfc->dmac);
2113 clk_disable_unprepare(nfc->mod_clk);
2114 clk_disable_unprepare(nfc->ahb_clk);
2115
2116 return 0;
2117 }
2118
2119 static const struct of_device_id sunxi_nfc_ids[] = {
2120 { .compatible = "allwinner,sun4i-a10-nand" },
2121 { /* sentinel */ }
2122 };
2123 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2124
2125 static struct platform_driver sunxi_nfc_driver = {
2126 .driver = {
2127 .name = "sunxi_nand",
2128 .of_match_table = sunxi_nfc_ids,
2129 },
2130 .probe = sunxi_nfc_probe,
2131 .remove = sunxi_nfc_remove,
2132 };
2133 module_platform_driver(sunxi_nfc_driver);
2134
2135 MODULE_LICENSE("GPL v2");
2136 MODULE_AUTHOR("Boris BREZILLON");
2137 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2138 MODULE_ALIAS("platform:sunxi_nand");
2139