81 	struct dm_rq_target_io *tio = info->tio;  in end_clone_bio()  local
88 	if (tio->error)  in end_clone_bio()
101 		tio->error = error;  in end_clone_bio()
109 	tio->completed += nr_bytes;  in end_clone_bio()
118 		blk_update_request(tio->orig, BLK_STS_OK, tio->completed);  in end_clone_bio()
129 		struct dm_rq_target_io *tio = tio_from_request(orig);  in rq_end_stats()  local
130 		tio->duration_jiffies = jiffies - tio->duration_jiffies;  in rq_end_stats()
132 				    blk_rq_pos(orig), tio->n_sectors, true,  in rq_end_stats()
133 				    tio->duration_jiffies, &tio->stats_aux);  in rq_end_stats()
157 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()  local
158 	struct mapped_device *md = tio->md;  in dm_end_request()
159 	struct request *rq = tio->orig;  in dm_end_request()
162 	tio->ti->type->release_clone_rq(clone, NULL);  in dm_end_request()
186 static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_requeue)  in dm_requeue_original_request()  argument
188 	struct mapped_device *md = tio->md;  in dm_requeue_original_request()
189 	struct request *rq = tio->orig;  in dm_requeue_original_request()
193 	if (tio->clone) {  in dm_requeue_original_request()
194 		blk_rq_unprep_clone(tio->clone);  in dm_requeue_original_request()
195 		tio->ti->type->release_clone_rq(tio->clone, NULL);  in dm_requeue_original_request()
205 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()  local
208 	if (tio->ti) {  in dm_done()
209 		rq_end_io = tio->ti->type->rq_end_io;  in dm_done()
212 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
218 			disable_discard(tio->md);  in dm_done()
221 			disable_write_zeroes(tio->md);  in dm_done()
234 		dm_requeue_original_request(tio, false);  in dm_done()
238 		dm_requeue_original_request(tio, true);  in dm_done()
252 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_softirq_done()  local
253 	struct request *clone = tio->clone;  in dm_softirq_done()
256 		struct mapped_device *md = tio->md;  in dm_softirq_done()
259 		blk_mq_end_request(rq, tio->error);  in dm_softirq_done()
267 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
276 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_complete_request()  local
278 	tio->error = error;  in dm_complete_request()
298 	struct dm_rq_target_io *tio = clone->end_io_data;  in end_clone_request()  local
300 	dm_complete_request(tio->orig, error);  in end_clone_request()
307 	struct dm_rq_target_io *tio = data;  in dm_rq_bio_constructor()  local
312 	info->tio = tio;  in dm_rq_bio_constructor()
319 		       struct dm_rq_target_io *tio, gfp_t gfp_mask)  in setup_clone()  argument
323 	r = blk_rq_prep_clone(clone, rq, &tio->md->mempools->bs, gfp_mask,  in setup_clone()
324 			      dm_rq_bio_constructor, tio);  in setup_clone()
329 	clone->end_io_data = tio;  in setup_clone()
331 	tio->clone = clone;  in setup_clone()
336 static void init_tio(struct dm_rq_target_io *tio, struct request *rq,  in init_tio()  argument
339 	tio->md = md;  in init_tio()
340 	tio->ti = NULL;  in init_tio()
341 	tio->clone = NULL;  in init_tio()
342 	tio->orig = rq;  in init_tio()
343 	tio->error = 0;  in init_tio()
344 	tio->completed = 0;  in init_tio()
351 		memset(&tio->info, 0, sizeof(tio->info));  in init_tio()
360 static int map_request(struct dm_rq_target_io *tio)  in map_request()  argument
363 	struct dm_target *ti = tio->ti;  in map_request()
364 	struct mapped_device *md = tio->md;  in map_request()
365 	struct request *rq = tio->orig;  in map_request()
369 	r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
375 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
377 			ti->type->release_clone_rq(clone, &tio->info);  in map_request()
392 			tio->ti->type->release_clone_rq(clone, &tio->info);  in map_request()
393 			tio->clone = NULL;  in map_request()
405 		dm_requeue_original_request(tio, true);  in map_request()
436 		struct dm_rq_target_io *tio = tio_from_request(orig);  in dm_start_request()  local
437 		tio->duration_jiffies = jiffies;  in dm_start_request()
438 		tio->n_sectors = blk_rq_sectors(orig);  in dm_start_request()
440 				    blk_rq_pos(orig), tio->n_sectors, false, 0,  in dm_start_request()
441 				    &tio->stats_aux);  in dm_start_request()
458 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_init_request()  local
464 	tio->md = md;  in dm_mq_init_request()
468 		tio->info.ptr = tio + 1;  in dm_mq_init_request()
478 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_queue_rq()  local
479 	struct mapped_device *md = tio->md;  in dm_mq_queue_rq()
509 	init_tio(tio, rq, md);  in dm_mq_queue_rq()
514 	tio->ti = ti;  in dm_mq_queue_rq()
517 	if (map_request(tio) == DM_MAPIO_REQUEUE) {  in dm_mq_queue_rq()