44  * @pa: page_array on which to perform the operation
56  *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova is not NULL
59 static int page_array_alloc(struct page_array *pa, u64 iova, unsigned int len)  in page_array_alloc()  argument
63 	if (pa->pa_nr || pa->pa_iova)  in page_array_alloc()
66 	pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;  in page_array_alloc()
67 	if (!pa->pa_nr)  in page_array_alloc()
70 	pa->pa_iova = kcalloc(pa->pa_nr,  in page_array_alloc()
71 			      sizeof(*pa->pa_iova) + sizeof(*pa->pa_page),  in page_array_alloc()
73 	if (unlikely(!pa->pa_iova)) {  in page_array_alloc()
74 		pa->pa_nr = 0;  in page_array_alloc()
77 	pa->pa_page = (struct page **)&pa->pa_iova[pa->pa_nr];  in page_array_alloc()
79 	pa->pa_iova[0] = iova;  in page_array_alloc()
80 	pa->pa_page[0] = NULL;  in page_array_alloc()
81 	for (i = 1; i < pa->pa_nr; i++) {  in page_array_alloc()
82 		pa->pa_iova[i] = pa->pa_iova[i - 1] + PAGE_SIZE;  in page_array_alloc()
83 		pa->pa_page[i] = NULL;  in page_array_alloc()
91  * @pa: page_array on which to perform the operation
96  * otherwise only clear pa->pa_nr
98 static void page_array_unpin(struct page_array *pa,  in page_array_unpin()  argument
104 		dma_addr_t *first = &pa->pa_iova[unpinned];  in page_array_unpin()
118 	pa->pa_nr = 0;  in page_array_unpin()
123  * @pa: page_array on which to perform the operation
130 static int page_array_pin(struct page_array *pa, struct vfio_device *vdev)  in page_array_pin()  argument
135 	while (pinned < pa->pa_nr) {  in page_array_pin()
136 		dma_addr_t *first = &pa->pa_iova[pinned];  in page_array_pin()
139 		if (pinned + npage < pa->pa_nr &&  in page_array_pin()
147 				     &pa->pa_page[pinned]);  in page_array_pin()
162 	page_array_unpin(pa, vdev, pinned);  in page_array_pin()
167 static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdev)  in page_array_unpin_free()  argument
169 	page_array_unpin(pa, vdev, pa->pa_nr);  in page_array_unpin_free()
170 	kfree(pa->pa_iova);  in page_array_unpin_free()
173 static bool page_array_iova_pinned(struct page_array *pa, u64 iova, u64 length)  in page_array_iova_pinned()  argument
180 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_iova_pinned()
181 		pfn = pa->pa_iova[i] >> PAGE_SHIFT;  in page_array_iova_pinned()
189 static inline void page_array_idal_create_words(struct page_array *pa,  in page_array_idal_create_words()  argument
202 	for (i = 0; i < pa->pa_nr; i++)  in page_array_idal_create_words()
203 		idaws[i] = page_to_phys(pa->pa_page[i]);  in page_array_idal_create_words()
206 	idaws[0] += pa->pa_iova[0] & (PAGE_SIZE - 1);  in page_array_idal_create_words()
238 	struct page_array pa = {0};  in copy_from_iova()  local
242 	ret = page_array_alloc(&pa, iova, n);  in copy_from_iova()
246 	ret = page_array_pin(&pa, vdev);  in copy_from_iova()
248 		page_array_unpin_free(&pa, vdev);  in copy_from_iova()
253 	for (i = 0; i < pa.pa_nr; i++) {  in copy_from_iova()
254 		void *from = kmap_local_page(pa.pa_page[i]);  in copy_from_iova()
271 	page_array_unpin_free(&pa, vdev);  in copy_from_iova()
558 	struct page_array *pa;  in ccwchain_fetch_direct()  local
597 	pa = chain->ch_pa + idx;  in ccwchain_fetch_direct()
598 	ret = page_array_alloc(pa, iova, bytes);  in ccwchain_fetch_direct()
613 			pa->pa_iova[i] = idaws[i];  in ccwchain_fetch_direct()
623 		ret = page_array_pin(pa, vdev);  in ccwchain_fetch_direct()
627 		pa->pa_nr = 0;  in ccwchain_fetch_direct()
634 	page_array_idal_create_words(pa, idaws);  in ccwchain_fetch_direct()
639 	page_array_unpin_free(pa, vdev);  in ccwchain_fetch_direct()