32  *    from this software without specific prior written permission.
60  * @from: source, must be 8 byte aligned
61  * @count: number of DWORD (32-bit) quantities to copy from source
63  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
72 	      const void *from, size_t count)  in pio_copy()  argument
92 			writeq(*(u64 *)from, dest);  in pio_copy()
93 			from += sizeof(u64);  in pio_copy()
108 			writeq(*(u64 *)from, dest);  in pio_copy()
109 			from += sizeof(u64);  in pio_copy()
127 				writeq(*(u64 *)from, dest);  in pio_copy()
128 				from += sizeof(u64);  in pio_copy()
138 			writeq(*(u64 *)from, dest);  in pio_copy()
139 			from += sizeof(u64);  in pio_copy()
150 		val.val32[0] = *(u32 *)from;  in pio_copy()
216  * Read nbytes from "from" and and place them in the low bytes
221  * o do not read from from if nbytes is zero
222  * o from may _not_ be u64 aligned.
224 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,  in read_low_bytes()  argument
228 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
233  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
237  * o from may _not_ be u64 aligned
241 				    const void *from, unsigned int nbytes)  in read_extra_bytes()  argument
243 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
300  * @from: data source, QWORD aligned
304 			const void *from, size_t nbytes)  in seg_pio_copy_start()  argument
323 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
324 			from += sizeof(u64);  in seg_pio_copy_start()
339 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
340 			from += sizeof(u64);  in seg_pio_copy_start()
358 				writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
359 				from += sizeof(u64);  in seg_pio_copy_start()
369 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
370 			from += sizeof(u64);  in seg_pio_copy_start()
379 	read_low_bytes(pbuf, from, nbytes & 0x7);  in seg_pio_copy_start()
391  * @from: data source, is QWORD aligned.
396 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)  in mid_copy_mix()  argument
428 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
429 			from += sizeof(u64);  in mid_copy_mix()
455 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
456 			from += sizeof(u64);  in mid_copy_mix()
466 		merge_write8(pbuf, dest, from);  in mid_copy_mix()
467 		from += sizeof(u64);  in mid_copy_mix()
479 		read_extra_bytes(pbuf, from, nread);  in mid_copy_mix()
505 		from += nread; /* from is now not aligned */  in mid_copy_mix()
506 		read_low_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
509 		read_extra_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
518  * @from: data source, is QWORD aligned
524 			      const void *from, size_t nbytes)  in mid_copy_straight()  argument
554 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
555 			from += sizeof(u64);  in mid_copy_straight()
581 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
582 			from += sizeof(u64);  in mid_copy_straight()
592 		writeq(*(u64 *)from, dest);  in mid_copy_straight()
593 		from += sizeof(u64);  in mid_copy_straight()
598 	read_low_bytes(pbuf, from, nbytes & 0x7);  in mid_copy_straight()
609  * @from: data source
612 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)  in seg_pio_copy_mid()  argument
614 	unsigned long from_align = (unsigned long)from & 0x7;  in seg_pio_copy_mid()
618 		read_extra_bytes(pbuf, from, nbytes);  in seg_pio_copy_mid()
626 		/* bytes to read to align "from" */  in seg_pio_copy_mid()
637 			read_extra_bytes(pbuf, from, to_align);  in seg_pio_copy_mid()
638 			from += to_align;  in seg_pio_copy_mid()
648 			read_extra_bytes(pbuf, from, to_fill);  in seg_pio_copy_mid()
649 			from += to_fill;  in seg_pio_copy_mid()
678 			read_low_bytes(pbuf, from, extra);  in seg_pio_copy_mid()
679 			from += extra;  in seg_pio_copy_mid()
684 			 * "extra" may have been reduced in size and "from"  in seg_pio_copy_mid()
692 		/* at this point, from is QW aligned */  in seg_pio_copy_mid()
696 		mid_copy_mix(pbuf, from, nbytes);  in seg_pio_copy_mid()
698 		mid_copy_straight(pbuf, from, nbytes);  in seg_pio_copy_mid()