1 /*
2  * Copyright (C) 2003 Sistina Software
3  * Copyright (C) 2006 Red Hat GmbH
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 
10 #include <linux/device-mapper.h>
11 
12 #include <linux/bio.h>
13 #include <linux/completion.h>
14 #include <linux/mempool.h>
15 #include <linux/module.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/dm-io.h>
19 
20 #define DM_MSG_PREFIX "io"
21 
22 #define DM_IO_MAX_REGIONS	BITS_PER_LONG
23 
24 struct dm_io_client {
25 	mempool_t pool;
26 	struct bio_set bios;
27 };
28 
29 /*
30  * Aligning 'struct io' reduces the number of bits required to store
31  * its address.  Refer to store_io_and_region_in_bio() below.
32  */
33 struct io {
34 	unsigned long error_bits;
35 	atomic_t count;
36 	struct dm_io_client *client;
37 	io_notify_fn callback;
38 	void *context;
39 	void *vma_invalidate_address;
40 	unsigned long vma_invalidate_size;
41 } __attribute__((aligned(DM_IO_MAX_REGIONS)));
42 
43 static struct kmem_cache *_dm_io_cache;
44 
45 /*
46  * Create a client with mempool and bioset.
47  */
dm_io_client_create(void)48 struct dm_io_client *dm_io_client_create(void)
49 {
50 	struct dm_io_client *client;
51 	unsigned min_ios = dm_get_reserved_bio_based_ios();
52 	int ret;
53 
54 	client = kzalloc(sizeof(*client), GFP_KERNEL);
55 	if (!client)
56 		return ERR_PTR(-ENOMEM);
57 
58 	ret = mempool_init_slab_pool(&client->pool, min_ios, _dm_io_cache);
59 	if (ret)
60 		goto bad;
61 
62 	ret = bioset_init(&client->bios, min_ios, 0, BIOSET_NEED_BVECS);
63 	if (ret)
64 		goto bad;
65 
66 	return client;
67 
68    bad:
69 	mempool_exit(&client->pool);
70 	kfree(client);
71 	return ERR_PTR(ret);
72 }
73 EXPORT_SYMBOL(dm_io_client_create);
74 
dm_io_client_destroy(struct dm_io_client * client)75 void dm_io_client_destroy(struct dm_io_client *client)
76 {
77 	mempool_exit(&client->pool);
78 	bioset_exit(&client->bios);
79 	kfree(client);
80 }
81 EXPORT_SYMBOL(dm_io_client_destroy);
82 
83 /*-----------------------------------------------------------------
84  * We need to keep track of which region a bio is doing io for.
85  * To avoid a memory allocation to store just 5 or 6 bits, we
86  * ensure the 'struct io' pointer is aligned so enough low bits are
87  * always zero and then combine it with the region number directly in
88  * bi_private.
89  *---------------------------------------------------------------*/
store_io_and_region_in_bio(struct bio * bio,struct io * io,unsigned region)90 static void store_io_and_region_in_bio(struct bio *bio, struct io *io,
91 				       unsigned region)
92 {
93 	if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) {
94 		DMCRIT("Unaligned struct io pointer %p", io);
95 		BUG();
96 	}
97 
98 	bio->bi_private = (void *)((unsigned long)io | region);
99 }
100 
retrieve_io_and_region_from_bio(struct bio * bio,struct io ** io,unsigned * region)101 static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,
102 				       unsigned *region)
103 {
104 	unsigned long val = (unsigned long)bio->bi_private;
105 
106 	*io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS);
107 	*region = val & (DM_IO_MAX_REGIONS - 1);
108 }
109 
110 /*-----------------------------------------------------------------
111  * We need an io object to keep track of the number of bios that
112  * have been dispatched for a particular io.
113  *---------------------------------------------------------------*/
complete_io(struct io * io)114 static void complete_io(struct io *io)
115 {
116 	unsigned long error_bits = io->error_bits;
117 	io_notify_fn fn = io->callback;
118 	void *context = io->context;
119 
120 	if (io->vma_invalidate_size)
121 		invalidate_kernel_vmap_range(io->vma_invalidate_address,
122 					     io->vma_invalidate_size);
123 
124 	mempool_free(io, &io->client->pool);
125 	fn(error_bits, context);
126 }
127 
dec_count(struct io * io,unsigned int region,blk_status_t error)128 static void dec_count(struct io *io, unsigned int region, blk_status_t error)
129 {
130 	if (error)
131 		set_bit(region, &io->error_bits);
132 
133 	if (atomic_dec_and_test(&io->count))
134 		complete_io(io);
135 }
136 
endio(struct bio * bio)137 static void endio(struct bio *bio)
138 {
139 	struct io *io;
140 	unsigned region;
141 	blk_status_t error;
142 
143 	if (bio->bi_status && bio_data_dir(bio) == READ)
144 		zero_fill_bio(bio);
145 
146 	/*
147 	 * The bio destructor in bio_put() may use the io object.
148 	 */
149 	retrieve_io_and_region_from_bio(bio, &io, &region);
150 
151 	error = bio->bi_status;
152 	bio_put(bio);
153 
154 	dec_count(io, region, error);
155 }
156 
157 /*-----------------------------------------------------------------
158  * These little objects provide an abstraction for getting a new
159  * destination page for io.
160  *---------------------------------------------------------------*/
161 struct dpages {
162 	void (*get_page)(struct dpages *dp,
163 			 struct page **p, unsigned long *len, unsigned *offset);
164 	void (*next_page)(struct dpages *dp);
165 
166 	union {
167 		unsigned context_u;
168 		struct bvec_iter context_bi;
169 	};
170 	void *context_ptr;
171 
172 	void *vma_invalidate_address;
173 	unsigned long vma_invalidate_size;
174 };
175 
176 /*
177  * Functions for getting the pages from a list.
178  */
list_get_page(struct dpages * dp,struct page ** p,unsigned long * len,unsigned * offset)179 static void list_get_page(struct dpages *dp,
180 		  struct page **p, unsigned long *len, unsigned *offset)
181 {
182 	unsigned o = dp->context_u;
183 	struct page_list *pl = (struct page_list *) dp->context_ptr;
184 
185 	*p = pl->page;
186 	*len = PAGE_SIZE - o;
187 	*offset = o;
188 }
189 
list_next_page(struct dpages * dp)190 static void list_next_page(struct dpages *dp)
191 {
192 	struct page_list *pl = (struct page_list *) dp->context_ptr;
193 	dp->context_ptr = pl->next;
194 	dp->context_u = 0;
195 }
196 
list_dp_init(struct dpages * dp,struct page_list * pl,unsigned offset)197 static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset)
198 {
199 	dp->get_page = list_get_page;
200 	dp->next_page = list_next_page;
201 	dp->context_u = offset;
202 	dp->context_ptr = pl;
203 }
204 
205 /*
206  * Functions for getting the pages from a bvec.
207  */
bio_get_page(struct dpages * dp,struct page ** p,unsigned long * len,unsigned * offset)208 static void bio_get_page(struct dpages *dp, struct page **p,
209 			 unsigned long *len, unsigned *offset)
210 {
211 	struct bio_vec bvec = bvec_iter_bvec((struct bio_vec *)dp->context_ptr,
212 					     dp->context_bi);
213 
214 	*p = bvec.bv_page;
215 	*len = bvec.bv_len;
216 	*offset = bvec.bv_offset;
217 
218 	/* avoid figuring it out again in bio_next_page() */
219 	dp->context_bi.bi_sector = (sector_t)bvec.bv_len;
220 }
221 
bio_next_page(struct dpages * dp)222 static void bio_next_page(struct dpages *dp)
223 {
224 	unsigned int len = (unsigned int)dp->context_bi.bi_sector;
225 
226 	bvec_iter_advance((struct bio_vec *)dp->context_ptr,
227 			  &dp->context_bi, len);
228 }
229 
bio_dp_init(struct dpages * dp,struct bio * bio)230 static void bio_dp_init(struct dpages *dp, struct bio *bio)
231 {
232 	dp->get_page = bio_get_page;
233 	dp->next_page = bio_next_page;
234 
235 	/*
236 	 * We just use bvec iterator to retrieve pages, so it is ok to
237 	 * access the bvec table directly here
238 	 */
239 	dp->context_ptr = bio->bi_io_vec;
240 	dp->context_bi = bio->bi_iter;
241 }
242 
243 /*
244  * Functions for getting the pages from a VMA.
245  */
vm_get_page(struct dpages * dp,struct page ** p,unsigned long * len,unsigned * offset)246 static void vm_get_page(struct dpages *dp,
247 		 struct page **p, unsigned long *len, unsigned *offset)
248 {
249 	*p = vmalloc_to_page(dp->context_ptr);
250 	*offset = dp->context_u;
251 	*len = PAGE_SIZE - dp->context_u;
252 }
253 
vm_next_page(struct dpages * dp)254 static void vm_next_page(struct dpages *dp)
255 {
256 	dp->context_ptr += PAGE_SIZE - dp->context_u;
257 	dp->context_u = 0;
258 }
259 
vm_dp_init(struct dpages * dp,void * data)260 static void vm_dp_init(struct dpages *dp, void *data)
261 {
262 	dp->get_page = vm_get_page;
263 	dp->next_page = vm_next_page;
264 	dp->context_u = offset_in_page(data);
265 	dp->context_ptr = data;
266 }
267 
268 /*
269  * Functions for getting the pages from kernel memory.
270  */
km_get_page(struct dpages * dp,struct page ** p,unsigned long * len,unsigned * offset)271 static void km_get_page(struct dpages *dp, struct page **p, unsigned long *len,
272 			unsigned *offset)
273 {
274 	*p = virt_to_page(dp->context_ptr);
275 	*offset = dp->context_u;
276 	*len = PAGE_SIZE - dp->context_u;
277 }
278 
km_next_page(struct dpages * dp)279 static void km_next_page(struct dpages *dp)
280 {
281 	dp->context_ptr += PAGE_SIZE - dp->context_u;
282 	dp->context_u = 0;
283 }
284 
km_dp_init(struct dpages * dp,void * data)285 static void km_dp_init(struct dpages *dp, void *data)
286 {
287 	dp->get_page = km_get_page;
288 	dp->next_page = km_next_page;
289 	dp->context_u = offset_in_page(data);
290 	dp->context_ptr = data;
291 }
292 
293 /*-----------------------------------------------------------------
294  * IO routines that accept a list of pages.
295  *---------------------------------------------------------------*/
do_region(const blk_opf_t opf,unsigned region,struct dm_io_region * where,struct dpages * dp,struct io * io)296 static void do_region(const blk_opf_t opf, unsigned region,
297 		      struct dm_io_region *where, struct dpages *dp,
298 		      struct io *io)
299 {
300 	struct bio *bio;
301 	struct page *page;
302 	unsigned long len;
303 	unsigned offset;
304 	unsigned num_bvecs;
305 	sector_t remaining = where->count;
306 	struct request_queue *q = bdev_get_queue(where->bdev);
307 	sector_t num_sectors;
308 	unsigned int special_cmd_max_sectors;
309 	const enum req_op op = opf & REQ_OP_MASK;
310 
311 	/*
312 	 * Reject unsupported discard and write same requests.
313 	 */
314 	if (op == REQ_OP_DISCARD)
315 		special_cmd_max_sectors = bdev_max_discard_sectors(where->bdev);
316 	else if (op == REQ_OP_WRITE_ZEROES)
317 		special_cmd_max_sectors = q->limits.max_write_zeroes_sectors;
318 	if ((op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) &&
319 	    special_cmd_max_sectors == 0) {
320 		atomic_inc(&io->count);
321 		dec_count(io, region, BLK_STS_NOTSUPP);
322 		return;
323 	}
324 
325 	/*
326 	 * where->count may be zero if op holds a flush and we need to
327 	 * send a zero-sized flush.
328 	 */
329 	do {
330 		/*
331 		 * Allocate a suitably sized-bio.
332 		 */
333 		switch (op) {
334 		case REQ_OP_DISCARD:
335 		case REQ_OP_WRITE_ZEROES:
336 			num_bvecs = 0;
337 			break;
338 		default:
339 			num_bvecs = bio_max_segs(dm_sector_div_up(remaining,
340 						(PAGE_SIZE >> SECTOR_SHIFT)));
341 		}
342 
343 		bio = bio_alloc_bioset(where->bdev, num_bvecs, opf, GFP_NOIO,
344 				       &io->client->bios);
345 		bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
346 		bio->bi_end_io = endio;
347 		store_io_and_region_in_bio(bio, io, region);
348 
349 		if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) {
350 			num_sectors = min_t(sector_t, special_cmd_max_sectors, remaining);
351 			bio->bi_iter.bi_size = num_sectors << SECTOR_SHIFT;
352 			remaining -= num_sectors;
353 		} else while (remaining) {
354 			/*
355 			 * Try and add as many pages as possible.
356 			 */
357 			dp->get_page(dp, &page, &len, &offset);
358 			len = min(len, to_bytes(remaining));
359 			if (!bio_add_page(bio, page, len, offset))
360 				break;
361 
362 			offset = 0;
363 			remaining -= to_sector(len);
364 			dp->next_page(dp);
365 		}
366 
367 		atomic_inc(&io->count);
368 		submit_bio(bio);
369 	} while (remaining);
370 }
371 
dispatch_io(blk_opf_t opf,unsigned int num_regions,struct dm_io_region * where,struct dpages * dp,struct io * io,int sync)372 static void dispatch_io(blk_opf_t opf, unsigned int num_regions,
373 			struct dm_io_region *where, struct dpages *dp,
374 			struct io *io, int sync)
375 {
376 	int i;
377 	struct dpages old_pages = *dp;
378 
379 	BUG_ON(num_regions > DM_IO_MAX_REGIONS);
380 
381 	if (sync)
382 		opf |= REQ_SYNC;
383 
384 	/*
385 	 * For multiple regions we need to be careful to rewind
386 	 * the dp object for each call to do_region.
387 	 */
388 	for (i = 0; i < num_regions; i++) {
389 		*dp = old_pages;
390 		if (where[i].count || (opf & REQ_PREFLUSH))
391 			do_region(opf, i, where + i, dp, io);
392 	}
393 
394 	/*
395 	 * Drop the extra reference that we were holding to avoid
396 	 * the io being completed too early.
397 	 */
398 	dec_count(io, 0, 0);
399 }
400 
401 struct sync_io {
402 	unsigned long error_bits;
403 	struct completion wait;
404 };
405 
sync_io_complete(unsigned long error,void * context)406 static void sync_io_complete(unsigned long error, void *context)
407 {
408 	struct sync_io *sio = context;
409 
410 	sio->error_bits = error;
411 	complete(&sio->wait);
412 }
413 
sync_io(struct dm_io_client * client,unsigned int num_regions,struct dm_io_region * where,blk_opf_t opf,struct dpages * dp,unsigned long * error_bits)414 static int sync_io(struct dm_io_client *client, unsigned int num_regions,
415 		   struct dm_io_region *where, blk_opf_t opf, struct dpages *dp,
416 		   unsigned long *error_bits)
417 {
418 	struct io *io;
419 	struct sync_io sio;
420 
421 	if (num_regions > 1 && !op_is_write(opf)) {
422 		WARN_ON(1);
423 		return -EIO;
424 	}
425 
426 	init_completion(&sio.wait);
427 
428 	io = mempool_alloc(&client->pool, GFP_NOIO);
429 	io->error_bits = 0;
430 	atomic_set(&io->count, 1); /* see dispatch_io() */
431 	io->client = client;
432 	io->callback = sync_io_complete;
433 	io->context = &sio;
434 
435 	io->vma_invalidate_address = dp->vma_invalidate_address;
436 	io->vma_invalidate_size = dp->vma_invalidate_size;
437 
438 	dispatch_io(opf, num_regions, where, dp, io, 1);
439 
440 	wait_for_completion_io(&sio.wait);
441 
442 	if (error_bits)
443 		*error_bits = sio.error_bits;
444 
445 	return sio.error_bits ? -EIO : 0;
446 }
447 
async_io(struct dm_io_client * client,unsigned int num_regions,struct dm_io_region * where,blk_opf_t opf,struct dpages * dp,io_notify_fn fn,void * context)448 static int async_io(struct dm_io_client *client, unsigned int num_regions,
449 		    struct dm_io_region *where, blk_opf_t opf,
450 		    struct dpages *dp, io_notify_fn fn, void *context)
451 {
452 	struct io *io;
453 
454 	if (num_regions > 1 && !op_is_write(opf)) {
455 		WARN_ON(1);
456 		fn(1, context);
457 		return -EIO;
458 	}
459 
460 	io = mempool_alloc(&client->pool, GFP_NOIO);
461 	io->error_bits = 0;
462 	atomic_set(&io->count, 1); /* see dispatch_io() */
463 	io->client = client;
464 	io->callback = fn;
465 	io->context = context;
466 
467 	io->vma_invalidate_address = dp->vma_invalidate_address;
468 	io->vma_invalidate_size = dp->vma_invalidate_size;
469 
470 	dispatch_io(opf, num_regions, where, dp, io, 0);
471 	return 0;
472 }
473 
dp_init(struct dm_io_request * io_req,struct dpages * dp,unsigned long size)474 static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
475 		   unsigned long size)
476 {
477 	/* Set up dpages based on memory type */
478 
479 	dp->vma_invalidate_address = NULL;
480 	dp->vma_invalidate_size = 0;
481 
482 	switch (io_req->mem.type) {
483 	case DM_IO_PAGE_LIST:
484 		list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset);
485 		break;
486 
487 	case DM_IO_BIO:
488 		bio_dp_init(dp, io_req->mem.ptr.bio);
489 		break;
490 
491 	case DM_IO_VMA:
492 		flush_kernel_vmap_range(io_req->mem.ptr.vma, size);
493 		if ((io_req->bi_opf & REQ_OP_MASK) == REQ_OP_READ) {
494 			dp->vma_invalidate_address = io_req->mem.ptr.vma;
495 			dp->vma_invalidate_size = size;
496 		}
497 		vm_dp_init(dp, io_req->mem.ptr.vma);
498 		break;
499 
500 	case DM_IO_KMEM:
501 		km_dp_init(dp, io_req->mem.ptr.addr);
502 		break;
503 
504 	default:
505 		return -EINVAL;
506 	}
507 
508 	return 0;
509 }
510 
dm_io(struct dm_io_request * io_req,unsigned num_regions,struct dm_io_region * where,unsigned long * sync_error_bits)511 int dm_io(struct dm_io_request *io_req, unsigned num_regions,
512 	  struct dm_io_region *where, unsigned long *sync_error_bits)
513 {
514 	int r;
515 	struct dpages dp;
516 
517 	r = dp_init(io_req, &dp, (unsigned long)where->count << SECTOR_SHIFT);
518 	if (r)
519 		return r;
520 
521 	if (!io_req->notify.fn)
522 		return sync_io(io_req->client, num_regions, where,
523 			       io_req->bi_opf, &dp, sync_error_bits);
524 
525 	return async_io(io_req->client, num_regions, where,
526 			io_req->bi_opf, &dp, io_req->notify.fn,
527 			io_req->notify.context);
528 }
529 EXPORT_SYMBOL(dm_io);
530 
dm_io_init(void)531 int __init dm_io_init(void)
532 {
533 	_dm_io_cache = KMEM_CACHE(io, 0);
534 	if (!_dm_io_cache)
535 		return -ENOMEM;
536 
537 	return 0;
538 }
539 
dm_io_exit(void)540 void dm_io_exit(void)
541 {
542 	kmem_cache_destroy(_dm_io_cache);
543 	_dm_io_cache = NULL;
544 }
545