1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
8 /* additive distance between non-SOP and SOP space */
10 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
15  * pio_copy - copy data block to MMIO space
19  * @from: source, must be 8 byte aligned
20  * @count: number of DWORD (32-bit) quantities to copy from source
27  * o pbuf->start always starts on a block boundary
33 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in pio_copy()
35 	void __iomem *dend;			/* 8-byte data end */  in pio_copy()
41 	/* calculate where the QWORD data ends - in SOP=1 space */  in pio_copy()
65 		/* write 8-byte SOP chunk data */  in pio_copy()
72 		dest -= SOP_DISTANCE;  in pio_copy()
73 		dend -= SOP_DISTANCE;  in pio_copy()
80 		 * the buffer wraps, then pbuf->end == dend == dest  in pio_copy()
84 		if (pbuf->end <= dend) {  in pio_copy()
85 			while (dest < pbuf->end) {  in pio_copy()
91 			dest -= pbuf->sc->size;  in pio_copy()
92 			dend -= pbuf->sc->size;  in pio_copy()
95 		/* write 8-byte non-SOP, non-wrap chunk data */  in pio_copy()
114 	 * fill in rest of block, no need to check pbuf->end  in pio_copy()
123 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in pio_copy()
134  * "zero" shift - bit shift used to zero out upper bytes.  Input is
137 #define zshift(x) (8 * (8 - (x)))
140  * "merge" shift - bit shift used to merge with carry bytes.  Input is
146  * Jump copy - no-loop copy for < 8 bytes.
176  * of pbuf->carry.  Other bytes are left as-is.  Any previous
177  * value in pbuf->carry is lost.
181  * o from may _not_ be u64 aligned.
186 	pbuf->carry.val64 = 0;  in read_low_bytes()
187 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
188 	pbuf->carry_bytes = nbytes;  in read_low_bytes()
192  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
196  * o from may _not_ be u64 aligned
202 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
203 	pbuf->carry_bytes += nbytes;  in read_extra_bytes()
207  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
209  * pbuf->carry with the upper bytes zeroed..
213  * o src must be u64 aligned
223 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
225 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
239  * Returns 0 on nothing written, non-zero on quad word written.
243 	if (pbuf->carry_bytes) {  in carry_write8()
245 		writeq(pbuf->carry.val64, dest);  in carry_write8()
253  * Segmented PIO Copy - start
259  * @from: data source, QWORD aligned
265 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in seg_pio_copy_start()
267 	void __iomem *dend;			/* 8-byte data end */  in seg_pio_copy_start()
272 	/* calculate where the QWORD data ends - in SOP=1 space */  in seg_pio_copy_start()
296 		/* write 8-byte SOP chunk data */  in seg_pio_copy_start()
303 		dest -= SOP_DISTANCE;  in seg_pio_copy_start()
304 		dend -= SOP_DISTANCE;  in seg_pio_copy_start()
311 		 * the buffer wraps, then pbuf->end == dend == dest  in seg_pio_copy_start()
315 		if (pbuf->end <= dend) {  in seg_pio_copy_start()
316 			while (dest < pbuf->end) {  in seg_pio_copy_start()
322 			dest -= pbuf->sc->size;  in seg_pio_copy_start()
323 			dend -= pbuf->sc->size;  in seg_pio_copy_start()
326 		/* write 8-byte non-SOP, non-wrap chunk data */  in seg_pio_copy_start()
340 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);  in seg_pio_copy_start()
344  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
345  * bytes are non-zero.
350  * @from: data source, is QWORD aligned.
357 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_mix()
358 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_mix()
362 	/* calculate 8-byte data end */  in mid_copy_mix()
365 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_mix()
378 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_mix()
385 		/* write 8-byte chunk data */  in mid_copy_mix()
393 		dest -= SOP_DISTANCE;  in mid_copy_mix()
397 	 * - at dend  in mid_copy_mix()
398 	 * - at the wrap  in mid_copy_mix()
409 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_mix()
412 	if (pbuf->end <= dend) {  in mid_copy_mix()
413 		while (dest < pbuf->end) {  in mid_copy_mix()
419 		dest -= pbuf->sc->size;  in mid_copy_mix()
420 		dend -= pbuf->sc->size;  in mid_copy_mix()
423 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_mix()
430 	pbuf->qw_written += qw_to_write;  in mid_copy_mix()
432 	/* handle carry and left-over bytes */  in mid_copy_mix()
433 	if (pbuf->carry_bytes + bytes_left >= 8) {  in mid_copy_mix()
436 		/* there is enough to fill another qw - fill carry */  in mid_copy_mix()
437 		nread = 8 - pbuf->carry_bytes;  in mid_copy_mix()
441 		 * One more write - but need to make sure dest is correct.  in mid_copy_mix()
452 		if (dest >= pbuf->end)  in mid_copy_mix()
453 			dest -= pbuf->sc->size;  in mid_copy_mix()
455 		else if (pbuf->qw_written < PIO_BLOCK_QWS)  in mid_copy_mix()
459 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
460 		pbuf->qw_written++;  in mid_copy_mix()
463 		bytes_left -= nread;  in mid_copy_mix()
464 		from += nread; /* from is now not aligned */  in mid_copy_mix()
473  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
477  * @from: data source, is QWORD aligned
485 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_straight()
486 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_straight()
488 	/* calculate 8-byte data end */  in mid_copy_straight()
491 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_straight()
504 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_straight()
511 		/* write 8-byte chunk data */  in mid_copy_straight()
519 		dest -= SOP_DISTANCE;  in mid_copy_straight()
523 	 * - at dend  in mid_copy_straight()
524 	 * - at the wrap  in mid_copy_straight()
535 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_straight()
538 	if (pbuf->end <= dend) {  in mid_copy_straight()
539 		while (dest < pbuf->end) {  in mid_copy_straight()
545 		dest -= pbuf->sc->size;  in mid_copy_straight()
546 		dend -= pbuf->sc->size;  in mid_copy_straight()
549 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_straight()
559 	pbuf->qw_written += nbytes >> 3;  in mid_copy_straight()
563  * Segmented PIO Copy - middle
565  * Must handle any aligned tail and any aligned source with any byte count.
575 	if (pbuf->carry_bytes + nbytes < 8) {  in seg_pio_copy_mid()
582 		/* misaligned source pointer - align it */  in seg_pio_copy_mid()
586 		to_align = 8 - from_align;  in seg_pio_copy_mid()
589 		 * In the advance-to-alignment logic below, we do not need  in seg_pio_copy_mid()
594 		if (pbuf->carry_bytes + to_align < 8) {  in seg_pio_copy_mid()
598 			nbytes -= to_align;  in seg_pio_copy_mid()
601 			unsigned long to_fill = 8 - pbuf->carry_bytes;  in seg_pio_copy_mid()
602 			/* bytes left over to be read */  in seg_pio_copy_mid()
603 			unsigned long extra = to_align - to_fill;  in seg_pio_copy_mid()
609 			nbytes -= to_fill;  in seg_pio_copy_mid()
610 			/* may not be enough valid bytes left to align */  in seg_pio_copy_mid()
615 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_mid()
626 			if (dest >= pbuf->end)  in seg_pio_copy_mid()
627 				dest -= pbuf->sc->size;  in seg_pio_copy_mid()
629 			else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_mid()
632 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
633 			pbuf->qw_written++;  in seg_pio_copy_mid()
636 			/* this will overwrite anything in pbuf->carry */  in seg_pio_copy_mid()
639 			nbytes -= extra;  in seg_pio_copy_mid()
641 			 * If no bytes are left, return early - we are done.  in seg_pio_copy_mid()
642 			 * NOTE: This short-circuit is *required* because  in seg_pio_copy_mid()
644 			 * is not aligned, as required when leaving this  in seg_pio_copy_mid()
651 		/* at this point, from is QW aligned */  in seg_pio_copy_mid()
654 	if (pbuf->carry_bytes)  in seg_pio_copy_mid()
661  * Segmented PIO Copy - end
663  * Write any remainder (in pbuf->carry) and finish writing the whole block.
669 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_end()
679 	if (dest >= pbuf->end)  in seg_pio_copy_end()
680 		dest -= pbuf->sc->size;  in seg_pio_copy_end()
682 	else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_end()
713 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in seg_pio_copy_end()