194 	struct drbd_peer_request *peer_req, *tmp;  in reclaim_finished_net_peer_reqs()  local
201 	list_for_each_entry_safe(peer_req, tmp, &device->net_ee, w.list) {  in reclaim_finished_net_peer_reqs()
202 		if (drbd_peer_req_has_active_page(peer_req))  in reclaim_finished_net_peer_reqs()
204 		list_move(&peer_req->w.list, to_be_freed);  in reclaim_finished_net_peer_reqs()
211 	struct drbd_peer_request *peer_req, *t;  in drbd_reclaim_net_peer_reqs()  local
216 	list_for_each_entry_safe(peer_req, t, &reclaimed, w.list)  in drbd_reclaim_net_peer_reqs()
217 		drbd_free_net_peer_req(device, peer_req);  in drbd_reclaim_net_peer_reqs()
362 	struct drbd_peer_request *peer_req;  in drbd_alloc_peer_req()  local
369 	peer_req = mempool_alloc(&drbd_ee_mempool, gfp_mask & ~__GFP_HIGHMEM);  in drbd_alloc_peer_req()
370 	if (!peer_req) {  in drbd_alloc_peer_req()
383 	memset(peer_req, 0, sizeof(*peer_req));  in drbd_alloc_peer_req()
384 	INIT_LIST_HEAD(&peer_req->w.list);  in drbd_alloc_peer_req()
385 	drbd_clear_interval(&peer_req->i);  in drbd_alloc_peer_req()
386 	peer_req->i.size = request_size;  in drbd_alloc_peer_req()
387 	peer_req->i.sector = sector;  in drbd_alloc_peer_req()
388 	peer_req->submit_jif = jiffies;  in drbd_alloc_peer_req()
389 	peer_req->peer_device = peer_device;  in drbd_alloc_peer_req()
390 	peer_req->pages = page;  in drbd_alloc_peer_req()
395 	peer_req->block_id = id;  in drbd_alloc_peer_req()
397 	return peer_req;  in drbd_alloc_peer_req()
400 	mempool_free(peer_req, &drbd_ee_mempool);  in drbd_alloc_peer_req()
404 void __drbd_free_peer_req(struct drbd_device *device, struct drbd_peer_request *peer_req,  in __drbd_free_peer_req()  argument
408 	if (peer_req->flags & EE_HAS_DIGEST)  in __drbd_free_peer_req()
409 		kfree(peer_req->digest);  in __drbd_free_peer_req()
410 	drbd_free_pages(device, peer_req->pages, is_net);  in __drbd_free_peer_req()
411 	D_ASSERT(device, atomic_read(&peer_req->pending_bios) == 0);  in __drbd_free_peer_req()
412 	D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in __drbd_free_peer_req()
413 	if (!expect(!(peer_req->flags & EE_CALL_AL_COMPLETE_IO))) {  in __drbd_free_peer_req()
414 		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;  in __drbd_free_peer_req()
415 		drbd_al_complete_io(device, &peer_req->i);  in __drbd_free_peer_req()
417 	mempool_free(peer_req, &drbd_ee_mempool);  in __drbd_free_peer_req()
423 	struct drbd_peer_request *peer_req, *t;  in drbd_free_peer_reqs()  local
431 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {  in drbd_free_peer_reqs()
432 		__drbd_free_peer_req(device, peer_req, is_net);  in drbd_free_peer_reqs()
445 	struct drbd_peer_request *peer_req, *t;  in drbd_finish_peer_reqs()  local
453 	list_for_each_entry_safe(peer_req, t, &reclaimed, w.list)  in drbd_finish_peer_reqs()
454 		drbd_free_net_peer_req(device, peer_req);  in drbd_finish_peer_reqs()
460 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {  in drbd_finish_peer_reqs()
464 		err2 = peer_req->w.cb(&peer_req->w, !!err);  in drbd_finish_peer_reqs()
467 		drbd_free_peer_req(device, peer_req);  in drbd_finish_peer_reqs()
1592 …drbd_issue_peer_discard_or_zero_out(struct drbd_device *device, struct drbd_peer_request *peer_req)  in drbd_issue_peer_discard_or_zero_out()  argument
1599 		peer_req->flags |= EE_ZEROOUT;  in drbd_issue_peer_discard_or_zero_out()
1601 	if (drbd_issue_discard_or_zero_out(device, peer_req->i.sector,  in drbd_issue_peer_discard_or_zero_out()
1602 	    peer_req->i.size >> 9, peer_req->flags & (EE_ZEROOUT|EE_TRIM)))  in drbd_issue_peer_discard_or_zero_out()
1603 		peer_req->flags |= EE_WAS_ERROR;  in drbd_issue_peer_discard_or_zero_out()
1604 	drbd_endio_write_sec_final(peer_req);  in drbd_issue_peer_discard_or_zero_out()
1608 				  struct drbd_peer_request *peer_req)  in drbd_issue_peer_wsame()  argument
1611 	sector_t s = peer_req->i.sector;  in drbd_issue_peer_wsame()
1612 	sector_t nr = peer_req->i.size >> 9;  in drbd_issue_peer_wsame()
1613 	if (blkdev_issue_write_same(bdev, s, nr, GFP_NOIO, peer_req->pages))  in drbd_issue_peer_wsame()
1614 		peer_req->flags |= EE_WAS_ERROR;  in drbd_issue_peer_wsame()
1615 	drbd_endio_write_sec_final(peer_req);  in drbd_issue_peer_wsame()
1637 			     struct drbd_peer_request *peer_req,  in drbd_submit_peer_request()  argument
1643 	struct page *page = peer_req->pages;  in drbd_submit_peer_request()
1644 	sector_t sector = peer_req->i.sector;  in drbd_submit_peer_request()
1645 	unsigned data_size = peer_req->i.size;  in drbd_submit_peer_request()
1656 	if (peer_req->flags & (EE_TRIM|EE_WRITE_SAME|EE_ZEROOUT)) {  in drbd_submit_peer_request()
1659 		conn_wait_active_ee_empty(peer_req->peer_device->connection);  in drbd_submit_peer_request()
1662 		peer_req->submit_jif = jiffies;  in drbd_submit_peer_request()
1663 		peer_req->flags |= EE_SUBMITTED;  in drbd_submit_peer_request()
1667 		if (list_empty(&peer_req->w.list)) {  in drbd_submit_peer_request()
1669 			list_add_tail(&peer_req->w.list, &device->active_ee);  in drbd_submit_peer_request()
1673 		if (peer_req->flags & (EE_TRIM|EE_ZEROOUT))  in drbd_submit_peer_request()
1674 			drbd_issue_peer_discard_or_zero_out(device, peer_req);  in drbd_submit_peer_request()
1676 			drbd_issue_peer_wsame(device, peer_req);  in drbd_submit_peer_request()
1698 	bio->bi_private = peer_req;  in drbd_submit_peer_request()
1716 	atomic_set(&peer_req->pending_bios, n_bios);  in drbd_submit_peer_request()
1718 	peer_req->submit_jif = jiffies;  in drbd_submit_peer_request()
1719 	peer_req->flags |= EE_SUBMITTED;  in drbd_submit_peer_request()
1739 					     struct drbd_peer_request *peer_req)  in drbd_remove_epoch_entry_interval()  argument
1741 	struct drbd_interval *i = &peer_req->i;  in drbd_remove_epoch_entry_interval()
1863 	struct drbd_peer_request *peer_req;  in read_in_block()  local
1932 	peer_req = drbd_alloc_peer_req(peer_device, id, sector, ds, data_size, GFP_NOIO);  in read_in_block()
1933 	if (!peer_req)  in read_in_block()
1936 	peer_req->flags |= EE_WRITE;  in read_in_block()
1938 		peer_req->flags |= EE_TRIM;  in read_in_block()
1939 		return peer_req;  in read_in_block()
1942 		peer_req->flags |= EE_ZEROOUT;  in read_in_block()
1943 		return peer_req;  in read_in_block()
1946 		peer_req->flags |= EE_WRITE_SAME;  in read_in_block()
1950 	page = peer_req->pages;  in read_in_block()
1961 			drbd_free_peer_req(device, peer_req);  in read_in_block()
1968 		drbd_csum_ee_size(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv, data_size);  in read_in_block()
1972 			drbd_free_peer_req(device, peer_req);  in read_in_block()
1977 	return peer_req;  in read_in_block()
2062 	struct drbd_peer_request *peer_req =  in e_end_resync_block()  local
2064 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_end_resync_block()
2066 	sector_t sector = peer_req->i.sector;  in e_end_resync_block()
2069 	D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in e_end_resync_block()
2071 	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {  in e_end_resync_block()
2072 		drbd_set_in_sync(device, sector, peer_req->i.size);  in e_end_resync_block()
2073 		err = drbd_send_ack(peer_device, P_RS_WRITE_ACK, peer_req);  in e_end_resync_block()
2076 		drbd_rs_failed_io(device, sector, peer_req->i.size);  in e_end_resync_block()
2078 		err  = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);  in e_end_resync_block()
2089 	struct drbd_peer_request *peer_req;  in recv_resync_read()  local
2091 	peer_req = read_in_block(peer_device, ID_SYNCER, sector, pi);  in recv_resync_read()
2092 	if (!peer_req)  in recv_resync_read()
2101 	peer_req->w.cb = e_end_resync_block;  in recv_resync_read()
2102 	peer_req->submit_jif = jiffies;  in recv_resync_read()
2105 	list_add_tail(&peer_req->w.list, &device->sync_ee);  in recv_resync_read()
2109 	if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE, 0,  in recv_resync_read()
2116 	list_del(&peer_req->w.list);  in recv_resync_read()
2119 	drbd_free_peer_req(device, peer_req);  in recv_resync_read()
2236 	struct drbd_peer_request *peer_req =  in e_end_block()  local
2238 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_end_block()
2240 	sector_t sector = peer_req->i.sector;  in e_end_block()
2243 	if (peer_req->flags & EE_SEND_WRITE_ACK) {  in e_end_block()
2244 		if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {  in e_end_block()
2247 				peer_req->flags & EE_MAY_SET_IN_SYNC) ?  in e_end_block()
2249 			err = drbd_send_ack(peer_device, pcmd, peer_req);  in e_end_block()
2251 				drbd_set_in_sync(device, sector, peer_req->i.size);  in e_end_block()
2253 			err = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);  in e_end_block()
2262 	if (peer_req->flags & EE_IN_INTERVAL_TREE) {  in e_end_block()
2264 		D_ASSERT(device, !drbd_interval_empty(&peer_req->i));  in e_end_block()
2265 		drbd_remove_epoch_entry_interval(device, peer_req);  in e_end_block()
2266 		if (peer_req->flags & EE_RESTART_REQUESTS)  in e_end_block()
2267 			restart_conflicting_writes(device, sector, peer_req->i.size);  in e_end_block()
2270 		D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in e_end_block()
2272 …drbd_may_finish_epoch(peer_device->connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0)…  in e_end_block()
2279 	struct drbd_peer_request *peer_req =  in e_send_ack()  local
2281 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_send_ack()
2284 	err = drbd_send_ack(peer_device, ack, peer_req);  in e_send_ack()
2297 	struct drbd_peer_request *peer_req =  in e_send_retry_write()  local
2299 	struct drbd_connection *connection = peer_req->peer_device->connection;  in e_send_retry_write()
2342 static bool overlapping_resync_write(struct drbd_device *device, struct drbd_peer_request *peer_req)  in overlapping_resync_write()  argument
2349 		if (overlaps(peer_req->i.sector, peer_req->i.size,  in overlapping_resync_write()
2477 				  struct drbd_peer_request *peer_req)  in handle_write_conflicts()  argument
2479 	struct drbd_connection *connection = peer_req->peer_device->connection;  in handle_write_conflicts()
2481 	sector_t sector = peer_req->i.sector;  in handle_write_conflicts()
2482 	const unsigned int size = peer_req->i.size;  in handle_write_conflicts()
2491 	drbd_insert_interval(&device->write_requests, &peer_req->i);  in handle_write_conflicts()
2495 		if (i == &peer_req->i)  in handle_write_conflicts()
2531 			peer_req->w.cb = superseded ? e_send_superseded :  in handle_write_conflicts()
2533 			list_add_tail(&peer_req->w.list, &device->done_ee);  in handle_write_conflicts()
2534 			queue_work(connection->ack_sender, &peer_req->peer_device->send_acks_work);  in handle_write_conflicts()
2573 			peer_req->flags |= EE_RESTART_REQUESTS;  in handle_write_conflicts()
2580 		drbd_remove_epoch_entry_interval(device, peer_req);  in handle_write_conflicts()
2591 	struct drbd_peer_request *peer_req;  in receive_Data()  local
2622 	peer_req = read_in_block(peer_device, p->block_id, sector, pi);  in receive_Data()
2623 	if (!peer_req) {  in receive_Data()
2628 	peer_req->w.cb = e_end_block;  in receive_Data()
2629 	peer_req->submit_jif = jiffies;  in receive_Data()
2630 	peer_req->flags |= EE_APPLICATION;  in receive_Data()
2636 		D_ASSERT(peer_device, peer_req->i.size > 0);  in receive_Data()
2638 		D_ASSERT(peer_device, peer_req->pages == NULL);  in receive_Data()
2642 			peer_req->flags |= EE_ZEROOUT;  in receive_Data()
2644 		D_ASSERT(peer_device, peer_req->i.size > 0);  in receive_Data()
2646 		D_ASSERT(peer_device, peer_req->pages == NULL);  in receive_Data()
2649 			peer_req->flags |= EE_TRIM;  in receive_Data()
2650 	} else if (peer_req->pages == NULL) {  in receive_Data()
2651 		D_ASSERT(device, peer_req->i.size == 0);  in receive_Data()
2656 		peer_req->flags |= EE_MAY_SET_IN_SYNC;  in receive_Data()
2659 	peer_req->epoch = connection->current_epoch;  in receive_Data()
2660 	atomic_inc(&peer_req->epoch->epoch_size);  in receive_Data()
2661 	atomic_inc(&peer_req->epoch->active);  in receive_Data()
2680 		peer_req->flags |= EE_SEND_WRITE_ACK;  in receive_Data()
2689 		drbd_send_ack(peer_device, P_RECV_ACK, peer_req);  in receive_Data()
2695 		peer_req->flags |= EE_IN_INTERVAL_TREE;  in receive_Data()
2700 		err = handle_write_conflicts(device, peer_req);  in receive_Data()
2717 	if ((peer_req->flags & (EE_TRIM|EE_WRITE_SAME|EE_ZEROOUT)) == 0)  in receive_Data()
2718 		list_add_tail(&peer_req->w.list, &device->active_ee);  in receive_Data()
2722 		wait_event(device->ee_wait, !overlapping_resync_write(device, peer_req));  in receive_Data()
2726 		drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);  in receive_Data()
2727 		peer_req->flags &= ~EE_MAY_SET_IN_SYNC;  in receive_Data()
2728 		drbd_al_begin_io(device, &peer_req->i);  in receive_Data()
2729 		peer_req->flags |= EE_CALL_AL_COMPLETE_IO;  in receive_Data()
2732 	err = drbd_submit_peer_request(device, peer_req, op, op_flags,  in receive_Data()
2740 	list_del(&peer_req->w.list);  in receive_Data()
2741 	drbd_remove_epoch_entry_interval(device, peer_req);  in receive_Data()
2743 	if (peer_req->flags & EE_CALL_AL_COMPLETE_IO) {  in receive_Data()
2744 		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;  in receive_Data()
2745 		drbd_al_complete_io(device, &peer_req->i);  in receive_Data()
2749 	drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP);  in receive_Data()
2751 	drbd_free_peer_req(device, peer_req);  in receive_Data()
2841 	struct drbd_peer_request *peer_req;  in receive_DataRequest()  local
2898 	peer_req = drbd_alloc_peer_req(peer_device, p->block_id, sector, size,  in receive_DataRequest()
2900 	if (!peer_req) {  in receive_DataRequest()
2907 		peer_req->w.cb = w_e_end_data_req;  in receive_DataRequest()
2910 		peer_req->flags |= EE_APPLICATION;  in receive_DataRequest()
2918 		peer_req->flags |= EE_RS_THIN_REQ;  in receive_DataRequest()
2921 		peer_req->w.cb = w_e_end_rsdata_req;  in receive_DataRequest()
2937 		peer_req->digest = di;  in receive_DataRequest()
2938 		peer_req->flags |= EE_HAS_DIGEST;  in receive_DataRequest()
2945 			peer_req->w.cb = w_e_end_csum_rs_req;  in receive_DataRequest()
2953 			peer_req->w.cb = w_e_end_ov_reply;  in receive_DataRequest()
2977 		peer_req->w.cb = w_e_end_ov_req;  in receive_DataRequest()
3013 	list_add_tail(&peer_req->w.list, &device->read_ee);  in receive_DataRequest()
3030 	if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0,  in receive_DataRequest()
3039 	list_del(&peer_req->w.list);  in receive_DataRequest()
3044 	drbd_free_peer_req(device, peer_req);  in receive_DataRequest()
5011 		struct drbd_peer_request *peer_req;  in receive_rs_deallocated()  local
5014 		peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector,  in receive_rs_deallocated()
5016 		if (!peer_req) {  in receive_rs_deallocated()
5021 		peer_req->w.cb = e_end_resync_block;  in receive_rs_deallocated()
5022 		peer_req->submit_jif = jiffies;  in receive_rs_deallocated()
5023 		peer_req->flags |= EE_TRIM;  in receive_rs_deallocated()
5026 		list_add_tail(&peer_req->w.list, &device->sync_ee);  in receive_rs_deallocated()
5030 		err = drbd_submit_peer_request(device, peer_req, op, 0, DRBD_FAULT_RS_WR);  in receive_rs_deallocated()
5034 			list_del(&peer_req->w.list);  in receive_rs_deallocated()
5037 			drbd_free_peer_req(device, peer_req);  in receive_rs_deallocated()