19  * @from: source, must be 8 byte aligned
20  * @count: number of DWORD (32-bit) quantities to copy from source
22  * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
31 	      const void *from, size_t count)  in pio_copy()  argument
51 			writeq(*(u64 *)from, dest);  in pio_copy()
52 			from += sizeof(u64);  in pio_copy()
67 			writeq(*(u64 *)from, dest);  in pio_copy()
68 			from += sizeof(u64);  in pio_copy()
86 				writeq(*(u64 *)from, dest);  in pio_copy()
87 				from += sizeof(u64);  in pio_copy()
97 			writeq(*(u64 *)from, dest);  in pio_copy()
98 			from += sizeof(u64);  in pio_copy()
109 		val.val32[0] = *(u32 *)from;  in pio_copy()
175  * Read nbytes from "from" and and place them in the low bytes
180  * o do not read from from if nbytes is zero
181  * o from may _not_ be u64 aligned.
183 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,  in read_low_bytes()  argument
187 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
192  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
196  * o from may _not_ be u64 aligned
200 				    const void *from, unsigned int nbytes)  in read_extra_bytes()  argument
202 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
259  * @from: data source, QWORD aligned
263 			const void *from, size_t nbytes)  in seg_pio_copy_start()  argument
282 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
283 			from += sizeof(u64);  in seg_pio_copy_start()
298 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
299 			from += sizeof(u64);  in seg_pio_copy_start()
317 				writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
318 				from += sizeof(u64);  in seg_pio_copy_start()
328 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
329 			from += sizeof(u64);  in seg_pio_copy_start()
338 	read_low_bytes(pbuf, from, nbytes & 0x7);  in seg_pio_copy_start()
350  * @from: data source, is QWORD aligned.
355 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)  in mid_copy_mix()  argument
387 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
388 			from += sizeof(u64);  in mid_copy_mix()
414 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
415 			from += sizeof(u64);  in mid_copy_mix()
425 		merge_write8(pbuf, dest, from);  in mid_copy_mix()
426 		from += sizeof(u64);  in mid_copy_mix()
438 		read_extra_bytes(pbuf, from, nread);  in mid_copy_mix()
464 		from += nread; /* from is now not aligned */  in mid_copy_mix()
465 		read_low_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
468 		read_extra_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
477  * @from: data source, is QWORD aligned
483 			      const void *from, size_t nbytes)  in mid_copy_straight()  argument
513 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
514 			from += sizeof(u64);  in mid_copy_straight()
540 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
541 			from += sizeof(u64);  in mid_copy_straight()
551 		writeq(*(u64 *)from, dest);  in mid_copy_straight()
552 		from += sizeof(u64);  in mid_copy_straight()
557 	read_low_bytes(pbuf, from, nbytes & 0x7);  in mid_copy_straight()
568  * @from: data source
571 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)  in seg_pio_copy_mid()  argument
573 	unsigned long from_align = (unsigned long)from & 0x7;  in seg_pio_copy_mid()
577 		read_extra_bytes(pbuf, from, nbytes);  in seg_pio_copy_mid()
585 		/* bytes to read to align "from" */  in seg_pio_copy_mid()
596 			read_extra_bytes(pbuf, from, to_align);  in seg_pio_copy_mid()
597 			from += to_align;  in seg_pio_copy_mid()
607 			read_extra_bytes(pbuf, from, to_fill);  in seg_pio_copy_mid()
608 			from += to_fill;  in seg_pio_copy_mid()
637 			read_low_bytes(pbuf, from, extra);  in seg_pio_copy_mid()
638 			from += extra;  in seg_pio_copy_mid()
643 			 * "extra" may have been reduced in size and "from"  in seg_pio_copy_mid()
651 		/* at this point, from is QW aligned */  in seg_pio_copy_mid()
655 		mid_copy_mix(pbuf, from, nbytes);  in seg_pio_copy_mid()
657 		mid_copy_straight(pbuf, from, nbytes);  in seg_pio_copy_mid()