197 	struct drbd_peer_request *peer_req, *tmp;  in reclaim_finished_net_peer_reqs()  local
204 	list_for_each_entry_safe(peer_req, tmp, &device->net_ee, w.list) {  in reclaim_finished_net_peer_reqs()
205 		if (drbd_peer_req_has_active_page(peer_req))  in reclaim_finished_net_peer_reqs()
207 		list_move(&peer_req->w.list, to_be_freed);  in reclaim_finished_net_peer_reqs()
214 	struct drbd_peer_request *peer_req, *t;  in drbd_reclaim_net_peer_reqs()  local
219 	list_for_each_entry_safe(peer_req, t, &reclaimed, w.list)  in drbd_reclaim_net_peer_reqs()
220 		drbd_free_net_peer_req(device, peer_req);  in drbd_reclaim_net_peer_reqs()
365 	struct drbd_peer_request *peer_req;  in drbd_alloc_peer_req()  local
372 	peer_req = mempool_alloc(&drbd_ee_mempool, gfp_mask & ~__GFP_HIGHMEM);  in drbd_alloc_peer_req()
373 	if (!peer_req) {  in drbd_alloc_peer_req()
386 	memset(peer_req, 0, sizeof(*peer_req));  in drbd_alloc_peer_req()
387 	INIT_LIST_HEAD(&peer_req->w.list);  in drbd_alloc_peer_req()
388 	drbd_clear_interval(&peer_req->i);  in drbd_alloc_peer_req()
389 	peer_req->i.size = request_size;  in drbd_alloc_peer_req()
390 	peer_req->i.sector = sector;  in drbd_alloc_peer_req()
391 	peer_req->submit_jif = jiffies;  in drbd_alloc_peer_req()
392 	peer_req->peer_device = peer_device;  in drbd_alloc_peer_req()
393 	peer_req->pages = page;  in drbd_alloc_peer_req()
398 	peer_req->block_id = id;  in drbd_alloc_peer_req()
400 	return peer_req;  in drbd_alloc_peer_req()
403 	mempool_free(peer_req, &drbd_ee_mempool);  in drbd_alloc_peer_req()
407 void __drbd_free_peer_req(struct drbd_device *device, struct drbd_peer_request *peer_req,  in __drbd_free_peer_req()  argument
411 	if (peer_req->flags & EE_HAS_DIGEST)  in __drbd_free_peer_req()
412 		kfree(peer_req->digest);  in __drbd_free_peer_req()
413 	drbd_free_pages(device, peer_req->pages, is_net);  in __drbd_free_peer_req()
414 	D_ASSERT(device, atomic_read(&peer_req->pending_bios) == 0);  in __drbd_free_peer_req()
415 	D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in __drbd_free_peer_req()
416 	if (!expect(!(peer_req->flags & EE_CALL_AL_COMPLETE_IO))) {  in __drbd_free_peer_req()
417 		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;  in __drbd_free_peer_req()
418 		drbd_al_complete_io(device, &peer_req->i);  in __drbd_free_peer_req()
420 	mempool_free(peer_req, &drbd_ee_mempool);  in __drbd_free_peer_req()
426 	struct drbd_peer_request *peer_req, *t;  in drbd_free_peer_reqs()  local
434 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {  in drbd_free_peer_reqs()
435 		__drbd_free_peer_req(device, peer_req, is_net);  in drbd_free_peer_reqs()
448 	struct drbd_peer_request *peer_req, *t;  in drbd_finish_peer_reqs()  local
456 	list_for_each_entry_safe(peer_req, t, &reclaimed, w.list)  in drbd_finish_peer_reqs()
457 		drbd_free_net_peer_req(device, peer_req);  in drbd_finish_peer_reqs()
463 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {  in drbd_finish_peer_reqs()
467 		err2 = peer_req->w.cb(&peer_req->w, !!err);  in drbd_finish_peer_reqs()
470 		drbd_free_peer_req(device, peer_req);  in drbd_finish_peer_reqs()
1591 …drbd_issue_peer_discard_or_zero_out(struct drbd_device *device, struct drbd_peer_request *peer_req)  in drbd_issue_peer_discard_or_zero_out()  argument
1598 		peer_req->flags |= EE_ZEROOUT;  in drbd_issue_peer_discard_or_zero_out()
1600 	if (drbd_issue_discard_or_zero_out(device, peer_req->i.sector,  in drbd_issue_peer_discard_or_zero_out()
1601 	    peer_req->i.size >> 9, peer_req->flags & (EE_ZEROOUT|EE_TRIM)))  in drbd_issue_peer_discard_or_zero_out()
1602 		peer_req->flags |= EE_WAS_ERROR;  in drbd_issue_peer_discard_or_zero_out()
1603 	drbd_endio_write_sec_final(peer_req);  in drbd_issue_peer_discard_or_zero_out()
1623 			     struct drbd_peer_request *peer_req,  in drbd_submit_peer_request()  argument
1628 	struct page *page = peer_req->pages;  in drbd_submit_peer_request()
1629 	sector_t sector = peer_req->i.sector;  in drbd_submit_peer_request()
1630 	unsigned int data_size = peer_req->i.size;  in drbd_submit_peer_request()
1640 	if (peer_req->flags & (EE_TRIM | EE_ZEROOUT)) {  in drbd_submit_peer_request()
1643 		conn_wait_active_ee_empty(peer_req->peer_device->connection);  in drbd_submit_peer_request()
1646 		peer_req->submit_jif = jiffies;  in drbd_submit_peer_request()
1647 		peer_req->flags |= EE_SUBMITTED;  in drbd_submit_peer_request()
1651 		if (list_empty(&peer_req->w.list)) {  in drbd_submit_peer_request()
1653 			list_add_tail(&peer_req->w.list, &device->active_ee);  in drbd_submit_peer_request()
1657 		drbd_issue_peer_discard_or_zero_out(device, peer_req);  in drbd_submit_peer_request()
1673 	bio->bi_private = peer_req;  in drbd_submit_peer_request()
1691 	atomic_set(&peer_req->pending_bios, n_bios);  in drbd_submit_peer_request()
1693 	peer_req->submit_jif = jiffies;  in drbd_submit_peer_request()
1694 	peer_req->flags |= EE_SUBMITTED;  in drbd_submit_peer_request()
1706 					     struct drbd_peer_request *peer_req)  in drbd_remove_epoch_entry_interval()  argument
1708 	struct drbd_interval *i = &peer_req->i;  in drbd_remove_epoch_entry_interval()
1830 	struct drbd_peer_request *peer_req;  in read_in_block()  local
1886 	peer_req = drbd_alloc_peer_req(peer_device, id, sector, ds, data_size, GFP_NOIO);  in read_in_block()
1887 	if (!peer_req)  in read_in_block()
1890 	peer_req->flags |= EE_WRITE;  in read_in_block()
1892 		peer_req->flags |= EE_TRIM;  in read_in_block()
1893 		return peer_req;  in read_in_block()
1896 		peer_req->flags |= EE_ZEROOUT;  in read_in_block()
1897 		return peer_req;  in read_in_block()
1902 	page = peer_req->pages;  in read_in_block()
1913 			drbd_free_peer_req(device, peer_req);  in read_in_block()
1920 		drbd_csum_ee_size(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv, data_size);  in read_in_block()
1924 			drbd_free_peer_req(device, peer_req);  in read_in_block()
1929 	return peer_req;  in read_in_block()
2014 	struct drbd_peer_request *peer_req =  in e_end_resync_block()  local
2016 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_end_resync_block()
2018 	sector_t sector = peer_req->i.sector;  in e_end_resync_block()
2021 	D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in e_end_resync_block()
2023 	if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {  in e_end_resync_block()
2024 		drbd_set_in_sync(device, sector, peer_req->i.size);  in e_end_resync_block()
2025 		err = drbd_send_ack(peer_device, P_RS_WRITE_ACK, peer_req);  in e_end_resync_block()
2028 		drbd_rs_failed_io(device, sector, peer_req->i.size);  in e_end_resync_block()
2030 		err  = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);  in e_end_resync_block()
2041 	struct drbd_peer_request *peer_req;  in recv_resync_read()  local
2043 	peer_req = read_in_block(peer_device, ID_SYNCER, sector, pi);  in recv_resync_read()
2044 	if (!peer_req)  in recv_resync_read()
2053 	peer_req->w.cb = e_end_resync_block;  in recv_resync_read()
2054 	peer_req->submit_jif = jiffies;  in recv_resync_read()
2057 	list_add_tail(&peer_req->w.list, &device->sync_ee);  in recv_resync_read()
2061 	if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE,  in recv_resync_read()
2068 	list_del(&peer_req->w.list);  in recv_resync_read()
2071 	drbd_free_peer_req(device, peer_req);  in recv_resync_read()
2185 	struct drbd_peer_request *peer_req =  in e_end_block()  local
2187 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_end_block()
2189 	sector_t sector = peer_req->i.sector;  in e_end_block()
2192 	if (peer_req->flags & EE_SEND_WRITE_ACK) {  in e_end_block()
2193 		if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) {  in e_end_block()
2196 				peer_req->flags & EE_MAY_SET_IN_SYNC) ?  in e_end_block()
2198 			err = drbd_send_ack(peer_device, pcmd, peer_req);  in e_end_block()
2200 				drbd_set_in_sync(device, sector, peer_req->i.size);  in e_end_block()
2202 			err = drbd_send_ack(peer_device, P_NEG_ACK, peer_req);  in e_end_block()
2211 	if (peer_req->flags & EE_IN_INTERVAL_TREE) {  in e_end_block()
2213 		D_ASSERT(device, !drbd_interval_empty(&peer_req->i));  in e_end_block()
2214 		drbd_remove_epoch_entry_interval(device, peer_req);  in e_end_block()
2215 		if (peer_req->flags & EE_RESTART_REQUESTS)  in e_end_block()
2216 			restart_conflicting_writes(device, sector, peer_req->i.size);  in e_end_block()
2219 		D_ASSERT(device, drbd_interval_empty(&peer_req->i));  in e_end_block()
2221 …drbd_may_finish_epoch(peer_device->connection, peer_req->epoch, EV_PUT + (cancel ? EV_CLEANUP : 0)…  in e_end_block()
2228 	struct drbd_peer_request *peer_req =  in e_send_ack()  local
2230 	struct drbd_peer_device *peer_device = peer_req->peer_device;  in e_send_ack()
2233 	err = drbd_send_ack(peer_device, ack, peer_req);  in e_send_ack()
2246 	struct drbd_peer_request *peer_req =  in e_send_retry_write()  local
2248 	struct drbd_connection *connection = peer_req->peer_device->connection;  in e_send_retry_write()
2291 static bool overlapping_resync_write(struct drbd_device *device, struct drbd_peer_request *peer_req)  in overlapping_resync_write()  argument
2298 		if (overlaps(peer_req->i.sector, peer_req->i.size,  in overlapping_resync_write()
2424 				  struct drbd_peer_request *peer_req)  in handle_write_conflicts()  argument
2426 	struct drbd_connection *connection = peer_req->peer_device->connection;  in handle_write_conflicts()
2428 	sector_t sector = peer_req->i.sector;  in handle_write_conflicts()
2429 	const unsigned int size = peer_req->i.size;  in handle_write_conflicts()
2438 	drbd_insert_interval(&device->write_requests, &peer_req->i);  in handle_write_conflicts()
2442 		if (i == &peer_req->i)  in handle_write_conflicts()
2478 			peer_req->w.cb = superseded ? e_send_superseded :  in handle_write_conflicts()
2480 			list_add_tail(&peer_req->w.list, &device->done_ee);  in handle_write_conflicts()
2481 			queue_work(connection->ack_sender, &peer_req->peer_device->send_acks_work);  in handle_write_conflicts()
2520 			peer_req->flags |= EE_RESTART_REQUESTS;  in handle_write_conflicts()
2527 		drbd_remove_epoch_entry_interval(device, peer_req);  in handle_write_conflicts()
2538 	struct drbd_peer_request *peer_req;  in receive_Data()  local
2570 	peer_req = read_in_block(peer_device, p->block_id, sector, pi);  in receive_Data()
2571 	if (!peer_req) {  in receive_Data()
2576 	peer_req->w.cb = e_end_block;  in receive_Data()
2577 	peer_req->submit_jif = jiffies;  in receive_Data()
2578 	peer_req->flags |= EE_APPLICATION;  in receive_Data()
2584 		D_ASSERT(peer_device, peer_req->i.size > 0);  in receive_Data()
2586 		D_ASSERT(peer_device, peer_req->pages == NULL);  in receive_Data()
2590 			peer_req->flags |= EE_ZEROOUT;  in receive_Data()
2592 		D_ASSERT(peer_device, peer_req->i.size > 0);  in receive_Data()
2594 		D_ASSERT(peer_device, peer_req->pages == NULL);  in receive_Data()
2597 			peer_req->flags |= EE_TRIM;  in receive_Data()
2598 	} else if (peer_req->pages == NULL) {  in receive_Data()
2599 		D_ASSERT(device, peer_req->i.size == 0);  in receive_Data()
2604 		peer_req->flags |= EE_MAY_SET_IN_SYNC;  in receive_Data()
2607 	peer_req->epoch = connection->current_epoch;  in receive_Data()
2608 	atomic_inc(&peer_req->epoch->epoch_size);  in receive_Data()
2609 	atomic_inc(&peer_req->epoch->active);  in receive_Data()
2628 		peer_req->flags |= EE_SEND_WRITE_ACK;  in receive_Data()
2637 		drbd_send_ack(peer_device, P_RECV_ACK, peer_req);  in receive_Data()
2643 		peer_req->flags |= EE_IN_INTERVAL_TREE;  in receive_Data()
2648 		err = handle_write_conflicts(device, peer_req);  in receive_Data()
2665 	if ((peer_req->flags & (EE_TRIM | EE_ZEROOUT)) == 0)  in receive_Data()
2666 		list_add_tail(&peer_req->w.list, &device->active_ee);  in receive_Data()
2670 		wait_event(device->ee_wait, !overlapping_resync_write(device, peer_req));  in receive_Data()
2674 		drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);  in receive_Data()
2675 		peer_req->flags &= ~EE_MAY_SET_IN_SYNC;  in receive_Data()
2676 		drbd_al_begin_io(device, &peer_req->i);  in receive_Data()
2677 		peer_req->flags |= EE_CALL_AL_COMPLETE_IO;  in receive_Data()
2680 	err = drbd_submit_peer_request(device, peer_req, op | op_flags,  in receive_Data()
2688 	list_del(&peer_req->w.list);  in receive_Data()
2689 	drbd_remove_epoch_entry_interval(device, peer_req);  in receive_Data()
2691 	if (peer_req->flags & EE_CALL_AL_COMPLETE_IO) {  in receive_Data()
2692 		peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;  in receive_Data()
2693 		drbd_al_complete_io(device, &peer_req->i);  in receive_Data()
2697 	drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP);  in receive_Data()
2699 	drbd_free_peer_req(device, peer_req);  in receive_Data()
2789 	struct drbd_peer_request *peer_req;  in receive_DataRequest()  local
2846 	peer_req = drbd_alloc_peer_req(peer_device, p->block_id, sector, size,  in receive_DataRequest()
2848 	if (!peer_req) {  in receive_DataRequest()
2855 		peer_req->w.cb = w_e_end_data_req;  in receive_DataRequest()
2858 		peer_req->flags |= EE_APPLICATION;  in receive_DataRequest()
2866 		peer_req->flags |= EE_RS_THIN_REQ;  in receive_DataRequest()
2869 		peer_req->w.cb = w_e_end_rsdata_req;  in receive_DataRequest()
2885 		peer_req->digest = di;  in receive_DataRequest()
2886 		peer_req->flags |= EE_HAS_DIGEST;  in receive_DataRequest()
2893 			peer_req->w.cb = w_e_end_csum_rs_req;  in receive_DataRequest()
2901 			peer_req->w.cb = w_e_end_ov_reply;  in receive_DataRequest()
2925 		peer_req->w.cb = w_e_end_ov_req;  in receive_DataRequest()
2961 	list_add_tail(&peer_req->w.list, &device->read_ee);  in receive_DataRequest()
2978 	if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ,  in receive_DataRequest()
2987 	list_del(&peer_req->w.list);  in receive_DataRequest()
2992 	drbd_free_peer_req(device, peer_req);  in receive_DataRequest()
4949 		struct drbd_peer_request *peer_req;  in receive_rs_deallocated()  local
4952 		peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector,  in receive_rs_deallocated()
4954 		if (!peer_req) {  in receive_rs_deallocated()
4959 		peer_req->w.cb = e_end_resync_block;  in receive_rs_deallocated()
4960 		peer_req->submit_jif = jiffies;  in receive_rs_deallocated()
4961 		peer_req->flags |= EE_TRIM;  in receive_rs_deallocated()
4964 		list_add_tail(&peer_req->w.list, &device->sync_ee);  in receive_rs_deallocated()
4968 		err = drbd_submit_peer_request(device, peer_req, op,  in receive_rs_deallocated()
4973 			list_del(&peer_req->w.list);  in receive_rs_deallocated()
4976 			drbd_free_peer_req(device, peer_req);  in receive_rs_deallocated()