262 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_write_complete()  local
263 	struct page *page = sio->bvec[0].bv_page;  in sio_write_complete()
266 	if (ret != sio->len) {  in sio_write_complete()
279 		for (p = 0; p < sio->pages; p++) {  in sio_write_complete()
280 			page = sio->bvec[p].bv_page;  in sio_write_complete()
285 		for (p = 0; p < sio->pages; p++)  in sio_write_complete()
286 			count_swpout_vm_event(sio->bvec[p].bv_page);  in sio_write_complete()
289 	for (p = 0; p < sio->pages; p++)  in sio_write_complete()
290 		end_page_writeback(sio->bvec[p].bv_page);  in sio_write_complete()
292 	mempool_free(sio, sio_pool);  in sio_write_complete()
297 	struct swap_iocb *sio = NULL;  in swap_writepage_fs()  local
305 		sio = *wbc->swap_plug;  in swap_writepage_fs()
306 	if (sio) {  in swap_writepage_fs()
307 		if (sio->iocb.ki_filp != swap_file ||  in swap_writepage_fs()
308 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_writepage_fs()
309 			swap_write_unplug(sio);  in swap_writepage_fs()
310 			sio = NULL;  in swap_writepage_fs()
313 	if (!sio) {  in swap_writepage_fs()
314 		sio = mempool_alloc(sio_pool, GFP_NOIO);  in swap_writepage_fs()
315 		init_sync_kiocb(&sio->iocb, swap_file);  in swap_writepage_fs()
316 		sio->iocb.ki_complete = sio_write_complete;  in swap_writepage_fs()
317 		sio->iocb.ki_pos = pos;  in swap_writepage_fs()
318 		sio->pages = 0;  in swap_writepage_fs()
319 		sio->len = 0;  in swap_writepage_fs()
321 	sio->bvec[sio->pages].bv_page = page;  in swap_writepage_fs()
322 	sio->bvec[sio->pages].bv_len = thp_size(page);  in swap_writepage_fs()
323 	sio->bvec[sio->pages].bv_offset = 0;  in swap_writepage_fs()
324 	sio->len += thp_size(page);  in swap_writepage_fs()
325 	sio->pages += 1;  in swap_writepage_fs()
326 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) {  in swap_writepage_fs()
327 		swap_write_unplug(sio);  in swap_writepage_fs()
328 		sio = NULL;  in swap_writepage_fs()
331 		*wbc->swap_plug = sio;  in swap_writepage_fs()
373 void swap_write_unplug(struct swap_iocb *sio)  in swap_write_unplug()  argument
376 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in swap_write_unplug()
379 	iov_iter_bvec(&from, WRITE, sio->bvec, sio->pages, sio->len);  in swap_write_unplug()
380 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in swap_write_unplug()
382 		sio_write_complete(&sio->iocb, ret);  in swap_write_unplug()
387 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_read_complete()  local
390 	if (ret == sio->len) {  in sio_read_complete()
391 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
392 			struct page *page = sio->bvec[p].bv_page;  in sio_read_complete()
397 		count_vm_events(PSWPIN, sio->pages);  in sio_read_complete()
399 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
400 			struct page *page = sio->bvec[p].bv_page;  in sio_read_complete()
408 	mempool_free(sio, sio_pool);  in sio_read_complete()
415 	struct swap_iocb *sio = NULL;  in swap_readpage_fs()  local
419 		sio = *plug;  in swap_readpage_fs()
420 	if (sio) {  in swap_readpage_fs()
421 		if (sio->iocb.ki_filp != sis->swap_file ||  in swap_readpage_fs()
422 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_readpage_fs()
423 			swap_read_unplug(sio);  in swap_readpage_fs()
424 			sio = NULL;  in swap_readpage_fs()
427 	if (!sio) {  in swap_readpage_fs()
428 		sio = mempool_alloc(sio_pool, GFP_KERNEL);  in swap_readpage_fs()
429 		init_sync_kiocb(&sio->iocb, sis->swap_file);  in swap_readpage_fs()
430 		sio->iocb.ki_pos = pos;  in swap_readpage_fs()
431 		sio->iocb.ki_complete = sio_read_complete;  in swap_readpage_fs()
432 		sio->pages = 0;  in swap_readpage_fs()
433 		sio->len = 0;  in swap_readpage_fs()
435 	sio->bvec[sio->pages].bv_page = page;  in swap_readpage_fs()
436 	sio->bvec[sio->pages].bv_len = thp_size(page);  in swap_readpage_fs()
437 	sio->bvec[sio->pages].bv_offset = 0;  in swap_readpage_fs()
438 	sio->len += thp_size(page);  in swap_readpage_fs()
439 	sio->pages += 1;  in swap_readpage_fs()
440 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {  in swap_readpage_fs()
441 		swap_read_unplug(sio);  in swap_readpage_fs()
442 		sio = NULL;  in swap_readpage_fs()
445 		*plug = sio;  in swap_readpage_fs()
527 void __swap_read_unplug(struct swap_iocb *sio)  in __swap_read_unplug()  argument
530 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in __swap_read_unplug()
533 	iov_iter_bvec(&from, READ, sio->bvec, sio->pages, sio->len);  in __swap_read_unplug()
534 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in __swap_read_unplug()
536 		sio_read_complete(&sio->iocb, ret);  in __swap_read_unplug()