795 	size_t loop_len;  in scif_rma_local_cpu_copy()  local
800 	loop_len = PAGE_SIZE - offset_in_page;  in scif_rma_local_cpu_copy()
802 	if (rem_len < loop_len)  in scif_rma_local_cpu_copy()
803 		loop_len = rem_len;  in scif_rma_local_cpu_copy()
805 	window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
809 		memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
811 		memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
813 	offset += loop_len;  in scif_rma_local_cpu_copy()
814 	temp += loop_len;  in scif_rma_local_cpu_copy()
815 	rem_len -= loop_len;  in scif_rma_local_cpu_copy()
825 		loop_len = min(PAGE_SIZE, rem_len);  in scif_rma_local_cpu_copy()
826 		window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
830 			memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
832 			memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
833 		offset	+= loop_len;  in scif_rma_local_cpu_copy()
834 		temp	+= loop_len;  in scif_rma_local_cpu_copy()
835 		rem_len	-= loop_len;  in scif_rma_local_cpu_copy()
874 	size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;  in scif_rma_list_dma_copy_unaligned()  local
894 		loop_len = L1_CACHE_BYTES - offset_in_ca;  in scif_rma_list_dma_copy_unaligned()
895 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_unaligned()
897 						  loop_len,  in scif_rma_list_dma_copy_unaligned()
904 						loop_len,  in scif_rma_list_dma_copy_unaligned()
906 						!(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
909 						  loop_len, work->ordered &&  in scif_rma_list_dma_copy_unaligned()
910 						  !(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
911 		iounmap_remote(window_virt_addr, loop_len, work);  in scif_rma_list_dma_copy_unaligned()
913 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
914 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
915 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
916 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
939 		loop_len = min(nr_contig_bytes, remaining_len);  in scif_rma_list_dma_copy_unaligned()
942 			    !(remaining_len - loop_len) &&  in scif_rma_list_dma_copy_unaligned()
943 			    loop_len != L1_CACHE_BYTES) {  in scif_rma_list_dma_copy_unaligned()
958 							    loop_len -  in scif_rma_list_dma_copy_unaligned()
971 				offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
972 				temp_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
973 				window_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
974 				remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
975 				loop_len = remaining_len;  in scif_rma_list_dma_copy_unaligned()
982 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
998 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1012 					window_dma_addr, loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1024 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
1025 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
1026 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
1027 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
1092 	size_t loop_len, remaining_len, src_contig_bytes = 0;  in _scif_rma_list_dma_copy_aligned()  local
1134 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in _scif_rma_list_dma_copy_aligned()
1135 		loop_len = min(loop_len, remaining_len);  in _scif_rma_list_dma_copy_aligned()
1136 		if (work->ordered && !(remaining_len - loop_len)) {  in _scif_rma_list_dma_copy_aligned()
1145 							 loop_len - 1,  in _scif_rma_list_dma_copy_aligned()
1156 			src_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1157 			dst_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1158 			src_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1159 			dst_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1160 			remaining_len -= (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1161 			loop_len = remaining_len;  in _scif_rma_list_dma_copy_aligned()
1165 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1178 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1189 		src_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1190 		dst_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1191 		remaining_len -= loop_len;  in _scif_rma_list_dma_copy_aligned()
1210 	size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;  in scif_rma_list_dma_copy_aligned()  local
1232 		loop_len = L1_CACHE_BYTES - src_cache_off;  in scif_rma_list_dma_copy_aligned()
1233 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1238 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1241 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1247 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1250 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1254 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1258 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1259 						remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1262 			scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1263 						  remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1266 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1268 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1269 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1270 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1271 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1301 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in scif_rma_list_dma_copy_aligned()
1302 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1304 		    !(remaining_len - loop_len)) {  in scif_rma_list_dma_copy_aligned()
1316 							 loop_len -  in scif_rma_list_dma_copy_aligned()
1329 			src_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1330 			dst_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1331 			src_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1332 			dst_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1333 			remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1334 			loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1339 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1353 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1365 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1366 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1367 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1371 		loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1393 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1396 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1403 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1406 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1410 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1415 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1419 						  loop_len, work->ordered);  in scif_rma_list_dma_copy_aligned()
1421 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1424 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1425 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1443 	size_t loop_len, remaining_len;  in scif_rma_list_cpu_copy()  local
1460 		loop_len = min(PAGE_SIZE -  in scif_rma_list_cpu_copy()
1466 						 loop_len);  in scif_rma_list_cpu_copy()
1469 						  loop_len,  in scif_rma_list_cpu_copy()
1479 						 loop_len);  in scif_rma_list_cpu_copy()
1482 						  loop_len,  in scif_rma_list_cpu_copy()
1487 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1493 			memcpy(dst_virt, src_virt, loop_len);  in scif_rma_list_cpu_copy()
1497 					    src_virt, loop_len);  in scif_rma_list_cpu_copy()
1501 					      loop_len);  in scif_rma_list_cpu_copy()
1504 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1507 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1509 		src_offset += loop_len;  in scif_rma_list_cpu_copy()
1510 		dst_offset += loop_len;  in scif_rma_list_cpu_copy()
1511 		remaining_len -= loop_len;  in scif_rma_list_cpu_copy()