1 /*
2  * Copyright (C) 2012 Red Hat. All rights reserved.
3  *
4  * This file is released under the GPL.
5  */
6 
7 #include "dm.h"
8 #include "dm-bio-prison-v2.h"
9 #include "dm-bio-record.h"
10 #include "dm-cache-metadata.h"
11 #include "dm-io-tracker.h"
12 
13 #include <linux/dm-io.h>
14 #include <linux/dm-kcopyd.h>
15 #include <linux/jiffies.h>
16 #include <linux/init.h>
17 #include <linux/mempool.h>
18 #include <linux/module.h>
19 #include <linux/rwsem.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 
23 #define DM_MSG_PREFIX "cache"
24 
25 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
26 	"A percentage of time allocated for copying to and/or from cache");
27 
28 /*----------------------------------------------------------------*/
29 
30 /*
31  * Glossary:
32  *
33  * oblock: index of an origin block
34  * cblock: index of a cache block
35  * promotion: movement of a block from origin to cache
36  * demotion: movement of a block from cache to origin
37  * migration: movement of a block between the origin and cache device,
38  *	      either direction
39  */
40 
41 /*----------------------------------------------------------------*/
42 
43 /*
44  * Represents a chunk of future work.  'input' allows continuations to pass
45  * values between themselves, typically error values.
46  */
47 struct continuation {
48 	struct work_struct ws;
49 	blk_status_t input;
50 };
51 
init_continuation(struct continuation * k,void (* fn)(struct work_struct *))52 static inline void init_continuation(struct continuation *k,
53 				     void (*fn)(struct work_struct *))
54 {
55 	INIT_WORK(&k->ws, fn);
56 	k->input = 0;
57 }
58 
queue_continuation(struct workqueue_struct * wq,struct continuation * k)59 static inline void queue_continuation(struct workqueue_struct *wq,
60 				      struct continuation *k)
61 {
62 	queue_work(wq, &k->ws);
63 }
64 
65 /*----------------------------------------------------------------*/
66 
67 /*
68  * The batcher collects together pieces of work that need a particular
69  * operation to occur before they can proceed (typically a commit).
70  */
71 struct batcher {
72 	/*
73 	 * The operation that everyone is waiting for.
74 	 */
75 	blk_status_t (*commit_op)(void *context);
76 	void *commit_context;
77 
78 	/*
79 	 * This is how bios should be issued once the commit op is complete
80 	 * (accounted_request).
81 	 */
82 	void (*issue_op)(struct bio *bio, void *context);
83 	void *issue_context;
84 
85 	/*
86 	 * Queued work gets put on here after commit.
87 	 */
88 	struct workqueue_struct *wq;
89 
90 	spinlock_t lock;
91 	struct list_head work_items;
92 	struct bio_list bios;
93 	struct work_struct commit_work;
94 
95 	bool commit_scheduled;
96 };
97 
__commit(struct work_struct * _ws)98 static void __commit(struct work_struct *_ws)
99 {
100 	struct batcher *b = container_of(_ws, struct batcher, commit_work);
101 	blk_status_t r;
102 	struct list_head work_items;
103 	struct work_struct *ws, *tmp;
104 	struct continuation *k;
105 	struct bio *bio;
106 	struct bio_list bios;
107 
108 	INIT_LIST_HEAD(&work_items);
109 	bio_list_init(&bios);
110 
111 	/*
112 	 * We have to grab these before the commit_op to avoid a race
113 	 * condition.
114 	 */
115 	spin_lock_irq(&b->lock);
116 	list_splice_init(&b->work_items, &work_items);
117 	bio_list_merge(&bios, &b->bios);
118 	bio_list_init(&b->bios);
119 	b->commit_scheduled = false;
120 	spin_unlock_irq(&b->lock);
121 
122 	r = b->commit_op(b->commit_context);
123 
124 	list_for_each_entry_safe(ws, tmp, &work_items, entry) {
125 		k = container_of(ws, struct continuation, ws);
126 		k->input = r;
127 		INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
128 		queue_work(b->wq, ws);
129 	}
130 
131 	while ((bio = bio_list_pop(&bios))) {
132 		if (r) {
133 			bio->bi_status = r;
134 			bio_endio(bio);
135 		} else
136 			b->issue_op(bio, b->issue_context);
137 	}
138 }
139 
batcher_init(struct batcher * b,blk_status_t (* commit_op)(void *),void * commit_context,void (* issue_op)(struct bio * bio,void *),void * issue_context,struct workqueue_struct * wq)140 static void batcher_init(struct batcher *b,
141 			 blk_status_t (*commit_op)(void *),
142 			 void *commit_context,
143 			 void (*issue_op)(struct bio *bio, void *),
144 			 void *issue_context,
145 			 struct workqueue_struct *wq)
146 {
147 	b->commit_op = commit_op;
148 	b->commit_context = commit_context;
149 	b->issue_op = issue_op;
150 	b->issue_context = issue_context;
151 	b->wq = wq;
152 
153 	spin_lock_init(&b->lock);
154 	INIT_LIST_HEAD(&b->work_items);
155 	bio_list_init(&b->bios);
156 	INIT_WORK(&b->commit_work, __commit);
157 	b->commit_scheduled = false;
158 }
159 
async_commit(struct batcher * b)160 static void async_commit(struct batcher *b)
161 {
162 	queue_work(b->wq, &b->commit_work);
163 }
164 
continue_after_commit(struct batcher * b,struct continuation * k)165 static void continue_after_commit(struct batcher *b, struct continuation *k)
166 {
167 	bool commit_scheduled;
168 
169 	spin_lock_irq(&b->lock);
170 	commit_scheduled = b->commit_scheduled;
171 	list_add_tail(&k->ws.entry, &b->work_items);
172 	spin_unlock_irq(&b->lock);
173 
174 	if (commit_scheduled)
175 		async_commit(b);
176 }
177 
178 /*
179  * Bios are errored if commit failed.
180  */
issue_after_commit(struct batcher * b,struct bio * bio)181 static void issue_after_commit(struct batcher *b, struct bio *bio)
182 {
183        bool commit_scheduled;
184 
185        spin_lock_irq(&b->lock);
186        commit_scheduled = b->commit_scheduled;
187        bio_list_add(&b->bios, bio);
188        spin_unlock_irq(&b->lock);
189 
190        if (commit_scheduled)
191 	       async_commit(b);
192 }
193 
194 /*
195  * Call this if some urgent work is waiting for the commit to complete.
196  */
schedule_commit(struct batcher * b)197 static void schedule_commit(struct batcher *b)
198 {
199 	bool immediate;
200 
201 	spin_lock_irq(&b->lock);
202 	immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
203 	b->commit_scheduled = true;
204 	spin_unlock_irq(&b->lock);
205 
206 	if (immediate)
207 		async_commit(b);
208 }
209 
210 /*
211  * There are a couple of places where we let a bio run, but want to do some
212  * work before calling its endio function.  We do this by temporarily
213  * changing the endio fn.
214  */
215 struct dm_hook_info {
216 	bio_end_io_t *bi_end_io;
217 };
218 
dm_hook_bio(struct dm_hook_info * h,struct bio * bio,bio_end_io_t * bi_end_io,void * bi_private)219 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
220 			bio_end_io_t *bi_end_io, void *bi_private)
221 {
222 	h->bi_end_io = bio->bi_end_io;
223 
224 	bio->bi_end_io = bi_end_io;
225 	bio->bi_private = bi_private;
226 }
227 
dm_unhook_bio(struct dm_hook_info * h,struct bio * bio)228 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
229 {
230 	bio->bi_end_io = h->bi_end_io;
231 }
232 
233 /*----------------------------------------------------------------*/
234 
235 #define MIGRATION_POOL_SIZE 128
236 #define COMMIT_PERIOD HZ
237 #define MIGRATION_COUNT_WINDOW 10
238 
239 /*
240  * The block size of the device holding cache data must be
241  * between 32KB and 1GB.
242  */
243 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
244 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
245 
246 enum cache_metadata_mode {
247 	CM_WRITE,		/* metadata may be changed */
248 	CM_READ_ONLY,		/* metadata may not be changed */
249 	CM_FAIL
250 };
251 
252 enum cache_io_mode {
253 	/*
254 	 * Data is written to cached blocks only.  These blocks are marked
255 	 * dirty.  If you lose the cache device you will lose data.
256 	 * Potential performance increase for both reads and writes.
257 	 */
258 	CM_IO_WRITEBACK,
259 
260 	/*
261 	 * Data is written to both cache and origin.  Blocks are never
262 	 * dirty.  Potential performance benfit for reads only.
263 	 */
264 	CM_IO_WRITETHROUGH,
265 
266 	/*
267 	 * A degraded mode useful for various cache coherency situations
268 	 * (eg, rolling back snapshots).  Reads and writes always go to the
269 	 * origin.  If a write goes to a cached oblock, then the cache
270 	 * block is invalidated.
271 	 */
272 	CM_IO_PASSTHROUGH
273 };
274 
275 struct cache_features {
276 	enum cache_metadata_mode mode;
277 	enum cache_io_mode io_mode;
278 	unsigned metadata_version;
279 	bool discard_passdown:1;
280 };
281 
282 struct cache_stats {
283 	atomic_t read_hit;
284 	atomic_t read_miss;
285 	atomic_t write_hit;
286 	atomic_t write_miss;
287 	atomic_t demotion;
288 	atomic_t promotion;
289 	atomic_t writeback;
290 	atomic_t copies_avoided;
291 	atomic_t cache_cell_clash;
292 	atomic_t commit_count;
293 	atomic_t discard_count;
294 };
295 
296 struct cache {
297 	struct dm_target *ti;
298 	spinlock_t lock;
299 
300 	/*
301 	 * Fields for converting from sectors to blocks.
302 	 */
303 	int sectors_per_block_shift;
304 	sector_t sectors_per_block;
305 
306 	struct dm_cache_metadata *cmd;
307 
308 	/*
309 	 * Metadata is written to this device.
310 	 */
311 	struct dm_dev *metadata_dev;
312 
313 	/*
314 	 * The slower of the two data devices.  Typically a spindle.
315 	 */
316 	struct dm_dev *origin_dev;
317 
318 	/*
319 	 * The faster of the two data devices.  Typically an SSD.
320 	 */
321 	struct dm_dev *cache_dev;
322 
323 	/*
324 	 * Size of the origin device in _complete_ blocks and native sectors.
325 	 */
326 	dm_oblock_t origin_blocks;
327 	sector_t origin_sectors;
328 
329 	/*
330 	 * Size of the cache device in blocks.
331 	 */
332 	dm_cblock_t cache_size;
333 
334 	/*
335 	 * Invalidation fields.
336 	 */
337 	spinlock_t invalidation_lock;
338 	struct list_head invalidation_requests;
339 
340 	sector_t migration_threshold;
341 	wait_queue_head_t migration_wait;
342 	atomic_t nr_allocated_migrations;
343 
344 	/*
345 	 * The number of in flight migrations that are performing
346 	 * background io. eg, promotion, writeback.
347 	 */
348 	atomic_t nr_io_migrations;
349 
350 	struct bio_list deferred_bios;
351 
352 	struct rw_semaphore quiesce_lock;
353 
354 	/*
355 	 * origin_blocks entries, discarded if set.
356 	 */
357 	dm_dblock_t discard_nr_blocks;
358 	unsigned long *discard_bitset;
359 	uint32_t discard_block_size; /* a power of 2 times sectors per block */
360 
361 	/*
362 	 * Rather than reconstructing the table line for the status we just
363 	 * save it and regurgitate.
364 	 */
365 	unsigned nr_ctr_args;
366 	const char **ctr_args;
367 
368 	struct dm_kcopyd_client *copier;
369 	struct work_struct deferred_bio_worker;
370 	struct work_struct migration_worker;
371 	struct workqueue_struct *wq;
372 	struct delayed_work waker;
373 	struct dm_bio_prison_v2 *prison;
374 
375 	/*
376 	 * cache_size entries, dirty if set
377 	 */
378 	unsigned long *dirty_bitset;
379 	atomic_t nr_dirty;
380 
381 	unsigned policy_nr_args;
382 	struct dm_cache_policy *policy;
383 
384 	/*
385 	 * Cache features such as write-through.
386 	 */
387 	struct cache_features features;
388 
389 	struct cache_stats stats;
390 
391 	bool need_tick_bio:1;
392 	bool sized:1;
393 	bool invalidate:1;
394 	bool commit_requested:1;
395 	bool loaded_mappings:1;
396 	bool loaded_discards:1;
397 
398 	struct rw_semaphore background_work_lock;
399 
400 	struct batcher committer;
401 	struct work_struct commit_ws;
402 
403 	struct dm_io_tracker tracker;
404 
405 	mempool_t migration_pool;
406 
407 	struct bio_set bs;
408 };
409 
410 struct per_bio_data {
411 	bool tick:1;
412 	unsigned req_nr:2;
413 	struct dm_bio_prison_cell_v2 *cell;
414 	struct dm_hook_info hook_info;
415 	sector_t len;
416 };
417 
418 struct dm_cache_migration {
419 	struct continuation k;
420 	struct cache *cache;
421 
422 	struct policy_work *op;
423 	struct bio *overwrite_bio;
424 	struct dm_bio_prison_cell_v2 *cell;
425 
426 	dm_cblock_t invalidate_cblock;
427 	dm_oblock_t invalidate_oblock;
428 };
429 
430 /*----------------------------------------------------------------*/
431 
writethrough_mode(struct cache * cache)432 static bool writethrough_mode(struct cache *cache)
433 {
434 	return cache->features.io_mode == CM_IO_WRITETHROUGH;
435 }
436 
writeback_mode(struct cache * cache)437 static bool writeback_mode(struct cache *cache)
438 {
439 	return cache->features.io_mode == CM_IO_WRITEBACK;
440 }
441 
passthrough_mode(struct cache * cache)442 static inline bool passthrough_mode(struct cache *cache)
443 {
444 	return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
445 }
446 
447 /*----------------------------------------------------------------*/
448 
wake_deferred_bio_worker(struct cache * cache)449 static void wake_deferred_bio_worker(struct cache *cache)
450 {
451 	queue_work(cache->wq, &cache->deferred_bio_worker);
452 }
453 
wake_migration_worker(struct cache * cache)454 static void wake_migration_worker(struct cache *cache)
455 {
456 	if (passthrough_mode(cache))
457 		return;
458 
459 	queue_work(cache->wq, &cache->migration_worker);
460 }
461 
462 /*----------------------------------------------------------------*/
463 
alloc_prison_cell(struct cache * cache)464 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
465 {
466 	return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);
467 }
468 
free_prison_cell(struct cache * cache,struct dm_bio_prison_cell_v2 * cell)469 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
470 {
471 	dm_bio_prison_free_cell_v2(cache->prison, cell);
472 }
473 
alloc_migration(struct cache * cache)474 static struct dm_cache_migration *alloc_migration(struct cache *cache)
475 {
476 	struct dm_cache_migration *mg;
477 
478 	mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);
479 
480 	memset(mg, 0, sizeof(*mg));
481 
482 	mg->cache = cache;
483 	atomic_inc(&cache->nr_allocated_migrations);
484 
485 	return mg;
486 }
487 
free_migration(struct dm_cache_migration * mg)488 static void free_migration(struct dm_cache_migration *mg)
489 {
490 	struct cache *cache = mg->cache;
491 
492 	if (atomic_dec_and_test(&cache->nr_allocated_migrations))
493 		wake_up(&cache->migration_wait);
494 
495 	mempool_free(mg, &cache->migration_pool);
496 }
497 
498 /*----------------------------------------------------------------*/
499 
oblock_succ(dm_oblock_t b)500 static inline dm_oblock_t oblock_succ(dm_oblock_t b)
501 {
502 	return to_oblock(from_oblock(b) + 1ull);
503 }
504 
build_key(dm_oblock_t begin,dm_oblock_t end,struct dm_cell_key_v2 * key)505 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
506 {
507 	key->virtual = 0;
508 	key->dev = 0;
509 	key->block_begin = from_oblock(begin);
510 	key->block_end = from_oblock(end);
511 }
512 
513 /*
514  * We have two lock levels.  Level 0, which is used to prevent WRITEs, and
515  * level 1 which prevents *both* READs and WRITEs.
516  */
517 #define WRITE_LOCK_LEVEL 0
518 #define READ_WRITE_LOCK_LEVEL 1
519 
lock_level(struct bio * bio)520 static unsigned lock_level(struct bio *bio)
521 {
522 	return bio_data_dir(bio) == WRITE ?
523 		WRITE_LOCK_LEVEL :
524 		READ_WRITE_LOCK_LEVEL;
525 }
526 
527 /*----------------------------------------------------------------
528  * Per bio data
529  *--------------------------------------------------------------*/
530 
get_per_bio_data(struct bio * bio)531 static struct per_bio_data *get_per_bio_data(struct bio *bio)
532 {
533 	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
534 	BUG_ON(!pb);
535 	return pb;
536 }
537 
init_per_bio_data(struct bio * bio)538 static struct per_bio_data *init_per_bio_data(struct bio *bio)
539 {
540 	struct per_bio_data *pb = get_per_bio_data(bio);
541 
542 	pb->tick = false;
543 	pb->req_nr = dm_bio_get_target_bio_nr(bio);
544 	pb->cell = NULL;
545 	pb->len = 0;
546 
547 	return pb;
548 }
549 
550 /*----------------------------------------------------------------*/
551 
defer_bio(struct cache * cache,struct bio * bio)552 static void defer_bio(struct cache *cache, struct bio *bio)
553 {
554 	spin_lock_irq(&cache->lock);
555 	bio_list_add(&cache->deferred_bios, bio);
556 	spin_unlock_irq(&cache->lock);
557 
558 	wake_deferred_bio_worker(cache);
559 }
560 
defer_bios(struct cache * cache,struct bio_list * bios)561 static void defer_bios(struct cache *cache, struct bio_list *bios)
562 {
563 	spin_lock_irq(&cache->lock);
564 	bio_list_merge(&cache->deferred_bios, bios);
565 	bio_list_init(bios);
566 	spin_unlock_irq(&cache->lock);
567 
568 	wake_deferred_bio_worker(cache);
569 }
570 
571 /*----------------------------------------------------------------*/
572 
bio_detain_shared(struct cache * cache,dm_oblock_t oblock,struct bio * bio)573 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
574 {
575 	bool r;
576 	struct per_bio_data *pb;
577 	struct dm_cell_key_v2 key;
578 	dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
579 	struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
580 
581 	cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
582 
583 	build_key(oblock, end, &key);
584 	r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
585 	if (!r) {
586 		/*
587 		 * Failed to get the lock.
588 		 */
589 		free_prison_cell(cache, cell_prealloc);
590 		return r;
591 	}
592 
593 	if (cell != cell_prealloc)
594 		free_prison_cell(cache, cell_prealloc);
595 
596 	pb = get_per_bio_data(bio);
597 	pb->cell = cell;
598 
599 	return r;
600 }
601 
602 /*----------------------------------------------------------------*/
603 
is_dirty(struct cache * cache,dm_cblock_t b)604 static bool is_dirty(struct cache *cache, dm_cblock_t b)
605 {
606 	return test_bit(from_cblock(b), cache->dirty_bitset);
607 }
608 
set_dirty(struct cache * cache,dm_cblock_t cblock)609 static void set_dirty(struct cache *cache, dm_cblock_t cblock)
610 {
611 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
612 		atomic_inc(&cache->nr_dirty);
613 		policy_set_dirty(cache->policy, cblock);
614 	}
615 }
616 
617 /*
618  * These two are called when setting after migrations to force the policy
619  * and dirty bitset to be in sync.
620  */
force_set_dirty(struct cache * cache,dm_cblock_t cblock)621 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
622 {
623 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
624 		atomic_inc(&cache->nr_dirty);
625 	policy_set_dirty(cache->policy, cblock);
626 }
627 
force_clear_dirty(struct cache * cache,dm_cblock_t cblock)628 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
629 {
630 	if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
631 		if (atomic_dec_return(&cache->nr_dirty) == 0)
632 			dm_table_event(cache->ti->table);
633 	}
634 
635 	policy_clear_dirty(cache->policy, cblock);
636 }
637 
638 /*----------------------------------------------------------------*/
639 
block_size_is_power_of_two(struct cache * cache)640 static bool block_size_is_power_of_two(struct cache *cache)
641 {
642 	return cache->sectors_per_block_shift >= 0;
643 }
644 
block_div(dm_block_t b,uint32_t n)645 static dm_block_t block_div(dm_block_t b, uint32_t n)
646 {
647 	do_div(b, n);
648 
649 	return b;
650 }
651 
oblocks_per_dblock(struct cache * cache)652 static dm_block_t oblocks_per_dblock(struct cache *cache)
653 {
654 	dm_block_t oblocks = cache->discard_block_size;
655 
656 	if (block_size_is_power_of_two(cache))
657 		oblocks >>= cache->sectors_per_block_shift;
658 	else
659 		oblocks = block_div(oblocks, cache->sectors_per_block);
660 
661 	return oblocks;
662 }
663 
oblock_to_dblock(struct cache * cache,dm_oblock_t oblock)664 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
665 {
666 	return to_dblock(block_div(from_oblock(oblock),
667 				   oblocks_per_dblock(cache)));
668 }
669 
set_discard(struct cache * cache,dm_dblock_t b)670 static void set_discard(struct cache *cache, dm_dblock_t b)
671 {
672 	BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
673 	atomic_inc(&cache->stats.discard_count);
674 
675 	spin_lock_irq(&cache->lock);
676 	set_bit(from_dblock(b), cache->discard_bitset);
677 	spin_unlock_irq(&cache->lock);
678 }
679 
clear_discard(struct cache * cache,dm_dblock_t b)680 static void clear_discard(struct cache *cache, dm_dblock_t b)
681 {
682 	spin_lock_irq(&cache->lock);
683 	clear_bit(from_dblock(b), cache->discard_bitset);
684 	spin_unlock_irq(&cache->lock);
685 }
686 
is_discarded(struct cache * cache,dm_dblock_t b)687 static bool is_discarded(struct cache *cache, dm_dblock_t b)
688 {
689 	int r;
690 	spin_lock_irq(&cache->lock);
691 	r = test_bit(from_dblock(b), cache->discard_bitset);
692 	spin_unlock_irq(&cache->lock);
693 
694 	return r;
695 }
696 
is_discarded_oblock(struct cache * cache,dm_oblock_t b)697 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
698 {
699 	int r;
700 	spin_lock_irq(&cache->lock);
701 	r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
702 		     cache->discard_bitset);
703 	spin_unlock_irq(&cache->lock);
704 
705 	return r;
706 }
707 
708 /*----------------------------------------------------------------
709  * Remapping
710  *--------------------------------------------------------------*/
remap_to_origin(struct cache * cache,struct bio * bio)711 static void remap_to_origin(struct cache *cache, struct bio *bio)
712 {
713 	bio_set_dev(bio, cache->origin_dev->bdev);
714 }
715 
remap_to_cache(struct cache * cache,struct bio * bio,dm_cblock_t cblock)716 static void remap_to_cache(struct cache *cache, struct bio *bio,
717 			   dm_cblock_t cblock)
718 {
719 	sector_t bi_sector = bio->bi_iter.bi_sector;
720 	sector_t block = from_cblock(cblock);
721 
722 	bio_set_dev(bio, cache->cache_dev->bdev);
723 	if (!block_size_is_power_of_two(cache))
724 		bio->bi_iter.bi_sector =
725 			(block * cache->sectors_per_block) +
726 			sector_div(bi_sector, cache->sectors_per_block);
727 	else
728 		bio->bi_iter.bi_sector =
729 			(block << cache->sectors_per_block_shift) |
730 			(bi_sector & (cache->sectors_per_block - 1));
731 }
732 
check_if_tick_bio_needed(struct cache * cache,struct bio * bio)733 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
734 {
735 	struct per_bio_data *pb;
736 
737 	spin_lock_irq(&cache->lock);
738 	if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
739 	    bio_op(bio) != REQ_OP_DISCARD) {
740 		pb = get_per_bio_data(bio);
741 		pb->tick = true;
742 		cache->need_tick_bio = false;
743 	}
744 	spin_unlock_irq(&cache->lock);
745 }
746 
remap_to_origin_clear_discard(struct cache * cache,struct bio * bio,dm_oblock_t oblock)747 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
748 					  dm_oblock_t oblock)
749 {
750 	// FIXME: check_if_tick_bio_needed() is called way too much through this interface
751 	check_if_tick_bio_needed(cache, bio);
752 	remap_to_origin(cache, bio);
753 	if (bio_data_dir(bio) == WRITE)
754 		clear_discard(cache, oblock_to_dblock(cache, oblock));
755 }
756 
remap_to_cache_dirty(struct cache * cache,struct bio * bio,dm_oblock_t oblock,dm_cblock_t cblock)757 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
758 				 dm_oblock_t oblock, dm_cblock_t cblock)
759 {
760 	check_if_tick_bio_needed(cache, bio);
761 	remap_to_cache(cache, bio, cblock);
762 	if (bio_data_dir(bio) == WRITE) {
763 		set_dirty(cache, cblock);
764 		clear_discard(cache, oblock_to_dblock(cache, oblock));
765 	}
766 }
767 
get_bio_block(struct cache * cache,struct bio * bio)768 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
769 {
770 	sector_t block_nr = bio->bi_iter.bi_sector;
771 
772 	if (!block_size_is_power_of_two(cache))
773 		(void) sector_div(block_nr, cache->sectors_per_block);
774 	else
775 		block_nr >>= cache->sectors_per_block_shift;
776 
777 	return to_oblock(block_nr);
778 }
779 
accountable_bio(struct cache * cache,struct bio * bio)780 static bool accountable_bio(struct cache *cache, struct bio *bio)
781 {
782 	return bio_op(bio) != REQ_OP_DISCARD;
783 }
784 
accounted_begin(struct cache * cache,struct bio * bio)785 static void accounted_begin(struct cache *cache, struct bio *bio)
786 {
787 	struct per_bio_data *pb;
788 
789 	if (accountable_bio(cache, bio)) {
790 		pb = get_per_bio_data(bio);
791 		pb->len = bio_sectors(bio);
792 		dm_iot_io_begin(&cache->tracker, pb->len);
793 	}
794 }
795 
accounted_complete(struct cache * cache,struct bio * bio)796 static void accounted_complete(struct cache *cache, struct bio *bio)
797 {
798 	struct per_bio_data *pb = get_per_bio_data(bio);
799 
800 	dm_iot_io_end(&cache->tracker, pb->len);
801 }
802 
accounted_request(struct cache * cache,struct bio * bio)803 static void accounted_request(struct cache *cache, struct bio *bio)
804 {
805 	accounted_begin(cache, bio);
806 	dm_submit_bio_remap(bio, NULL);
807 }
808 
issue_op(struct bio * bio,void * context)809 static void issue_op(struct bio *bio, void *context)
810 {
811 	struct cache *cache = context;
812 	accounted_request(cache, bio);
813 }
814 
815 /*
816  * When running in writethrough mode we need to send writes to clean blocks
817  * to both the cache and origin devices.  Clone the bio and send them in parallel.
818  */
remap_to_origin_and_cache(struct cache * cache,struct bio * bio,dm_oblock_t oblock,dm_cblock_t cblock)819 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
820 				      dm_oblock_t oblock, dm_cblock_t cblock)
821 {
822 	struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio,
823 						 GFP_NOIO, &cache->bs);
824 
825 	BUG_ON(!origin_bio);
826 
827 	bio_chain(origin_bio, bio);
828 
829 	if (bio_data_dir(origin_bio) == WRITE)
830 		clear_discard(cache, oblock_to_dblock(cache, oblock));
831 	submit_bio(origin_bio);
832 
833 	remap_to_cache(cache, bio, cblock);
834 }
835 
836 /*----------------------------------------------------------------
837  * Failure modes
838  *--------------------------------------------------------------*/
get_cache_mode(struct cache * cache)839 static enum cache_metadata_mode get_cache_mode(struct cache *cache)
840 {
841 	return cache->features.mode;
842 }
843 
cache_device_name(struct cache * cache)844 static const char *cache_device_name(struct cache *cache)
845 {
846 	return dm_table_device_name(cache->ti->table);
847 }
848 
notify_mode_switch(struct cache * cache,enum cache_metadata_mode mode)849 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
850 {
851 	const char *descs[] = {
852 		"write",
853 		"read-only",
854 		"fail"
855 	};
856 
857 	dm_table_event(cache->ti->table);
858 	DMINFO("%s: switching cache to %s mode",
859 	       cache_device_name(cache), descs[(int)mode]);
860 }
861 
set_cache_mode(struct cache * cache,enum cache_metadata_mode new_mode)862 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
863 {
864 	bool needs_check;
865 	enum cache_metadata_mode old_mode = get_cache_mode(cache);
866 
867 	if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
868 		DMERR("%s: unable to read needs_check flag, setting failure mode.",
869 		      cache_device_name(cache));
870 		new_mode = CM_FAIL;
871 	}
872 
873 	if (new_mode == CM_WRITE && needs_check) {
874 		DMERR("%s: unable to switch cache to write mode until repaired.",
875 		      cache_device_name(cache));
876 		if (old_mode != new_mode)
877 			new_mode = old_mode;
878 		else
879 			new_mode = CM_READ_ONLY;
880 	}
881 
882 	/* Never move out of fail mode */
883 	if (old_mode == CM_FAIL)
884 		new_mode = CM_FAIL;
885 
886 	switch (new_mode) {
887 	case CM_FAIL:
888 	case CM_READ_ONLY:
889 		dm_cache_metadata_set_read_only(cache->cmd);
890 		break;
891 
892 	case CM_WRITE:
893 		dm_cache_metadata_set_read_write(cache->cmd);
894 		break;
895 	}
896 
897 	cache->features.mode = new_mode;
898 
899 	if (new_mode != old_mode)
900 		notify_mode_switch(cache, new_mode);
901 }
902 
abort_transaction(struct cache * cache)903 static void abort_transaction(struct cache *cache)
904 {
905 	const char *dev_name = cache_device_name(cache);
906 
907 	if (get_cache_mode(cache) >= CM_READ_ONLY)
908 		return;
909 
910 	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
911 	if (dm_cache_metadata_abort(cache->cmd)) {
912 		DMERR("%s: failed to abort metadata transaction", dev_name);
913 		set_cache_mode(cache, CM_FAIL);
914 	}
915 
916 	if (dm_cache_metadata_set_needs_check(cache->cmd)) {
917 		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
918 		set_cache_mode(cache, CM_FAIL);
919 	}
920 }
921 
metadata_operation_failed(struct cache * cache,const char * op,int r)922 static void metadata_operation_failed(struct cache *cache, const char *op, int r)
923 {
924 	DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
925 		    cache_device_name(cache), op, r);
926 	abort_transaction(cache);
927 	set_cache_mode(cache, CM_READ_ONLY);
928 }
929 
930 /*----------------------------------------------------------------*/
931 
load_stats(struct cache * cache)932 static void load_stats(struct cache *cache)
933 {
934 	struct dm_cache_statistics stats;
935 
936 	dm_cache_metadata_get_stats(cache->cmd, &stats);
937 	atomic_set(&cache->stats.read_hit, stats.read_hits);
938 	atomic_set(&cache->stats.read_miss, stats.read_misses);
939 	atomic_set(&cache->stats.write_hit, stats.write_hits);
940 	atomic_set(&cache->stats.write_miss, stats.write_misses);
941 }
942 
save_stats(struct cache * cache)943 static void save_stats(struct cache *cache)
944 {
945 	struct dm_cache_statistics stats;
946 
947 	if (get_cache_mode(cache) >= CM_READ_ONLY)
948 		return;
949 
950 	stats.read_hits = atomic_read(&cache->stats.read_hit);
951 	stats.read_misses = atomic_read(&cache->stats.read_miss);
952 	stats.write_hits = atomic_read(&cache->stats.write_hit);
953 	stats.write_misses = atomic_read(&cache->stats.write_miss);
954 
955 	dm_cache_metadata_set_stats(cache->cmd, &stats);
956 }
957 
update_stats(struct cache_stats * stats,enum policy_operation op)958 static void update_stats(struct cache_stats *stats, enum policy_operation op)
959 {
960 	switch (op) {
961 	case POLICY_PROMOTE:
962 		atomic_inc(&stats->promotion);
963 		break;
964 
965 	case POLICY_DEMOTE:
966 		atomic_inc(&stats->demotion);
967 		break;
968 
969 	case POLICY_WRITEBACK:
970 		atomic_inc(&stats->writeback);
971 		break;
972 	}
973 }
974 
975 /*----------------------------------------------------------------
976  * Migration processing
977  *
978  * Migration covers moving data from the origin device to the cache, or
979  * vice versa.
980  *--------------------------------------------------------------*/
981 
inc_io_migrations(struct cache * cache)982 static void inc_io_migrations(struct cache *cache)
983 {
984 	atomic_inc(&cache->nr_io_migrations);
985 }
986 
dec_io_migrations(struct cache * cache)987 static void dec_io_migrations(struct cache *cache)
988 {
989 	atomic_dec(&cache->nr_io_migrations);
990 }
991 
discard_or_flush(struct bio * bio)992 static bool discard_or_flush(struct bio *bio)
993 {
994 	return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
995 }
996 
calc_discard_block_range(struct cache * cache,struct bio * bio,dm_dblock_t * b,dm_dblock_t * e)997 static void calc_discard_block_range(struct cache *cache, struct bio *bio,
998 				     dm_dblock_t *b, dm_dblock_t *e)
999 {
1000 	sector_t sb = bio->bi_iter.bi_sector;
1001 	sector_t se = bio_end_sector(bio);
1002 
1003 	*b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
1004 
1005 	if (se - sb < cache->discard_block_size)
1006 		*e = *b;
1007 	else
1008 		*e = to_dblock(block_div(se, cache->discard_block_size));
1009 }
1010 
1011 /*----------------------------------------------------------------*/
1012 
prevent_background_work(struct cache * cache)1013 static void prevent_background_work(struct cache *cache)
1014 {
1015 	lockdep_off();
1016 	down_write(&cache->background_work_lock);
1017 	lockdep_on();
1018 }
1019 
allow_background_work(struct cache * cache)1020 static void allow_background_work(struct cache *cache)
1021 {
1022 	lockdep_off();
1023 	up_write(&cache->background_work_lock);
1024 	lockdep_on();
1025 }
1026 
background_work_begin(struct cache * cache)1027 static bool background_work_begin(struct cache *cache)
1028 {
1029 	bool r;
1030 
1031 	lockdep_off();
1032 	r = down_read_trylock(&cache->background_work_lock);
1033 	lockdep_on();
1034 
1035 	return r;
1036 }
1037 
background_work_end(struct cache * cache)1038 static void background_work_end(struct cache *cache)
1039 {
1040 	lockdep_off();
1041 	up_read(&cache->background_work_lock);
1042 	lockdep_on();
1043 }
1044 
1045 /*----------------------------------------------------------------*/
1046 
bio_writes_complete_block(struct cache * cache,struct bio * bio)1047 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1048 {
1049 	return (bio_data_dir(bio) == WRITE) &&
1050 		(bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1051 }
1052 
optimisable_bio(struct cache * cache,struct bio * bio,dm_oblock_t block)1053 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1054 {
1055 	return writeback_mode(cache) &&
1056 		(is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1057 }
1058 
quiesce(struct dm_cache_migration * mg,void (* continuation)(struct work_struct *))1059 static void quiesce(struct dm_cache_migration *mg,
1060 		    void (*continuation)(struct work_struct *))
1061 {
1062 	init_continuation(&mg->k, continuation);
1063 	dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1064 }
1065 
ws_to_mg(struct work_struct * ws)1066 static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1067 {
1068 	struct continuation *k = container_of(ws, struct continuation, ws);
1069 	return container_of(k, struct dm_cache_migration, k);
1070 }
1071 
copy_complete(int read_err,unsigned long write_err,void * context)1072 static void copy_complete(int read_err, unsigned long write_err, void *context)
1073 {
1074 	struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1075 
1076 	if (read_err || write_err)
1077 		mg->k.input = BLK_STS_IOERR;
1078 
1079 	queue_continuation(mg->cache->wq, &mg->k);
1080 }
1081 
copy(struct dm_cache_migration * mg,bool promote)1082 static void copy(struct dm_cache_migration *mg, bool promote)
1083 {
1084 	struct dm_io_region o_region, c_region;
1085 	struct cache *cache = mg->cache;
1086 
1087 	o_region.bdev = cache->origin_dev->bdev;
1088 	o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1089 	o_region.count = cache->sectors_per_block;
1090 
1091 	c_region.bdev = cache->cache_dev->bdev;
1092 	c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1093 	c_region.count = cache->sectors_per_block;
1094 
1095 	if (promote)
1096 		dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1097 	else
1098 		dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1099 }
1100 
bio_drop_shared_lock(struct cache * cache,struct bio * bio)1101 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1102 {
1103 	struct per_bio_data *pb = get_per_bio_data(bio);
1104 
1105 	if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1106 		free_prison_cell(cache, pb->cell);
1107 	pb->cell = NULL;
1108 }
1109 
overwrite_endio(struct bio * bio)1110 static void overwrite_endio(struct bio *bio)
1111 {
1112 	struct dm_cache_migration *mg = bio->bi_private;
1113 	struct cache *cache = mg->cache;
1114 	struct per_bio_data *pb = get_per_bio_data(bio);
1115 
1116 	dm_unhook_bio(&pb->hook_info, bio);
1117 
1118 	if (bio->bi_status)
1119 		mg->k.input = bio->bi_status;
1120 
1121 	queue_continuation(cache->wq, &mg->k);
1122 }
1123 
overwrite(struct dm_cache_migration * mg,void (* continuation)(struct work_struct *))1124 static void overwrite(struct dm_cache_migration *mg,
1125 		      void (*continuation)(struct work_struct *))
1126 {
1127 	struct bio *bio = mg->overwrite_bio;
1128 	struct per_bio_data *pb = get_per_bio_data(bio);
1129 
1130 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1131 
1132 	/*
1133 	 * The overwrite bio is part of the copy operation, as such it does
1134 	 * not set/clear discard or dirty flags.
1135 	 */
1136 	if (mg->op->op == POLICY_PROMOTE)
1137 		remap_to_cache(mg->cache, bio, mg->op->cblock);
1138 	else
1139 		remap_to_origin(mg->cache, bio);
1140 
1141 	init_continuation(&mg->k, continuation);
1142 	accounted_request(mg->cache, bio);
1143 }
1144 
1145 /*
1146  * Migration steps:
1147  *
1148  * 1) exclusive lock preventing WRITEs
1149  * 2) quiesce
1150  * 3) copy or issue overwrite bio
1151  * 4) upgrade to exclusive lock preventing READs and WRITEs
1152  * 5) quiesce
1153  * 6) update metadata and commit
1154  * 7) unlock
1155  */
mg_complete(struct dm_cache_migration * mg,bool success)1156 static void mg_complete(struct dm_cache_migration *mg, bool success)
1157 {
1158 	struct bio_list bios;
1159 	struct cache *cache = mg->cache;
1160 	struct policy_work *op = mg->op;
1161 	dm_cblock_t cblock = op->cblock;
1162 
1163 	if (success)
1164 		update_stats(&cache->stats, op->op);
1165 
1166 	switch (op->op) {
1167 	case POLICY_PROMOTE:
1168 		clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1169 		policy_complete_background_work(cache->policy, op, success);
1170 
1171 		if (mg->overwrite_bio) {
1172 			if (success)
1173 				force_set_dirty(cache, cblock);
1174 			else if (mg->k.input)
1175 				mg->overwrite_bio->bi_status = mg->k.input;
1176 			else
1177 				mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1178 			bio_endio(mg->overwrite_bio);
1179 		} else {
1180 			if (success)
1181 				force_clear_dirty(cache, cblock);
1182 			dec_io_migrations(cache);
1183 		}
1184 		break;
1185 
1186 	case POLICY_DEMOTE:
1187 		/*
1188 		 * We clear dirty here to update the nr_dirty counter.
1189 		 */
1190 		if (success)
1191 			force_clear_dirty(cache, cblock);
1192 		policy_complete_background_work(cache->policy, op, success);
1193 		dec_io_migrations(cache);
1194 		break;
1195 
1196 	case POLICY_WRITEBACK:
1197 		if (success)
1198 			force_clear_dirty(cache, cblock);
1199 		policy_complete_background_work(cache->policy, op, success);
1200 		dec_io_migrations(cache);
1201 		break;
1202 	}
1203 
1204 	bio_list_init(&bios);
1205 	if (mg->cell) {
1206 		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1207 			free_prison_cell(cache, mg->cell);
1208 	}
1209 
1210 	free_migration(mg);
1211 	defer_bios(cache, &bios);
1212 	wake_migration_worker(cache);
1213 
1214 	background_work_end(cache);
1215 }
1216 
mg_success(struct work_struct * ws)1217 static void mg_success(struct work_struct *ws)
1218 {
1219 	struct dm_cache_migration *mg = ws_to_mg(ws);
1220 	mg_complete(mg, mg->k.input == 0);
1221 }
1222 
mg_update_metadata(struct work_struct * ws)1223 static void mg_update_metadata(struct work_struct *ws)
1224 {
1225 	int r;
1226 	struct dm_cache_migration *mg = ws_to_mg(ws);
1227 	struct cache *cache = mg->cache;
1228 	struct policy_work *op = mg->op;
1229 
1230 	switch (op->op) {
1231 	case POLICY_PROMOTE:
1232 		r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1233 		if (r) {
1234 			DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1235 				    cache_device_name(cache));
1236 			metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1237 
1238 			mg_complete(mg, false);
1239 			return;
1240 		}
1241 		mg_complete(mg, true);
1242 		break;
1243 
1244 	case POLICY_DEMOTE:
1245 		r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1246 		if (r) {
1247 			DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1248 				    cache_device_name(cache));
1249 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1250 
1251 			mg_complete(mg, false);
1252 			return;
1253 		}
1254 
1255 		/*
1256 		 * It would be nice if we only had to commit when a REQ_FLUSH
1257 		 * comes through.  But there's one scenario that we have to
1258 		 * look out for:
1259 		 *
1260 		 * - vblock x in a cache block
1261 		 * - domotion occurs
1262 		 * - cache block gets reallocated and over written
1263 		 * - crash
1264 		 *
1265 		 * When we recover, because there was no commit the cache will
1266 		 * rollback to having the data for vblock x in the cache block.
1267 		 * But the cache block has since been overwritten, so it'll end
1268 		 * up pointing to data that was never in 'x' during the history
1269 		 * of the device.
1270 		 *
1271 		 * To avoid this issue we require a commit as part of the
1272 		 * demotion operation.
1273 		 */
1274 		init_continuation(&mg->k, mg_success);
1275 		continue_after_commit(&cache->committer, &mg->k);
1276 		schedule_commit(&cache->committer);
1277 		break;
1278 
1279 	case POLICY_WRITEBACK:
1280 		mg_complete(mg, true);
1281 		break;
1282 	}
1283 }
1284 
mg_update_metadata_after_copy(struct work_struct * ws)1285 static void mg_update_metadata_after_copy(struct work_struct *ws)
1286 {
1287 	struct dm_cache_migration *mg = ws_to_mg(ws);
1288 
1289 	/*
1290 	 * Did the copy succeed?
1291 	 */
1292 	if (mg->k.input)
1293 		mg_complete(mg, false);
1294 	else
1295 		mg_update_metadata(ws);
1296 }
1297 
mg_upgrade_lock(struct work_struct * ws)1298 static void mg_upgrade_lock(struct work_struct *ws)
1299 {
1300 	int r;
1301 	struct dm_cache_migration *mg = ws_to_mg(ws);
1302 
1303 	/*
1304 	 * Did the copy succeed?
1305 	 */
1306 	if (mg->k.input)
1307 		mg_complete(mg, false);
1308 
1309 	else {
1310 		/*
1311 		 * Now we want the lock to prevent both reads and writes.
1312 		 */
1313 		r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1314 					    READ_WRITE_LOCK_LEVEL);
1315 		if (r < 0)
1316 			mg_complete(mg, false);
1317 
1318 		else if (r)
1319 			quiesce(mg, mg_update_metadata);
1320 
1321 		else
1322 			mg_update_metadata(ws);
1323 	}
1324 }
1325 
mg_full_copy(struct work_struct * ws)1326 static void mg_full_copy(struct work_struct *ws)
1327 {
1328 	struct dm_cache_migration *mg = ws_to_mg(ws);
1329 	struct cache *cache = mg->cache;
1330 	struct policy_work *op = mg->op;
1331 	bool is_policy_promote = (op->op == POLICY_PROMOTE);
1332 
1333 	if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1334 	    is_discarded_oblock(cache, op->oblock)) {
1335 		mg_upgrade_lock(ws);
1336 		return;
1337 	}
1338 
1339 	init_continuation(&mg->k, mg_upgrade_lock);
1340 	copy(mg, is_policy_promote);
1341 }
1342 
mg_copy(struct work_struct * ws)1343 static void mg_copy(struct work_struct *ws)
1344 {
1345 	struct dm_cache_migration *mg = ws_to_mg(ws);
1346 
1347 	if (mg->overwrite_bio) {
1348 		/*
1349 		 * No exclusive lock was held when we last checked if the bio
1350 		 * was optimisable.  So we have to check again in case things
1351 		 * have changed (eg, the block may no longer be discarded).
1352 		 */
1353 		if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1354 			/*
1355 			 * Fallback to a real full copy after doing some tidying up.
1356 			 */
1357 			bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
1358 			BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */
1359 			mg->overwrite_bio = NULL;
1360 			inc_io_migrations(mg->cache);
1361 			mg_full_copy(ws);
1362 			return;
1363 		}
1364 
1365 		/*
1366 		 * It's safe to do this here, even though it's new data
1367 		 * because all IO has been locked out of the block.
1368 		 *
1369 		 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1370 		 * so _not_ using mg_upgrade_lock() as continutation.
1371 		 */
1372 		overwrite(mg, mg_update_metadata_after_copy);
1373 
1374 	} else
1375 		mg_full_copy(ws);
1376 }
1377 
mg_lock_writes(struct dm_cache_migration * mg)1378 static int mg_lock_writes(struct dm_cache_migration *mg)
1379 {
1380 	int r;
1381 	struct dm_cell_key_v2 key;
1382 	struct cache *cache = mg->cache;
1383 	struct dm_bio_prison_cell_v2 *prealloc;
1384 
1385 	prealloc = alloc_prison_cell(cache);
1386 
1387 	/*
1388 	 * Prevent writes to the block, but allow reads to continue.
1389 	 * Unless we're using an overwrite bio, in which case we lock
1390 	 * everything.
1391 	 */
1392 	build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1393 	r = dm_cell_lock_v2(cache->prison, &key,
1394 			    mg->overwrite_bio ?  READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1395 			    prealloc, &mg->cell);
1396 	if (r < 0) {
1397 		free_prison_cell(cache, prealloc);
1398 		mg_complete(mg, false);
1399 		return r;
1400 	}
1401 
1402 	if (mg->cell != prealloc)
1403 		free_prison_cell(cache, prealloc);
1404 
1405 	if (r == 0)
1406 		mg_copy(&mg->k.ws);
1407 	else
1408 		quiesce(mg, mg_copy);
1409 
1410 	return 0;
1411 }
1412 
mg_start(struct cache * cache,struct policy_work * op,struct bio * bio)1413 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1414 {
1415 	struct dm_cache_migration *mg;
1416 
1417 	if (!background_work_begin(cache)) {
1418 		policy_complete_background_work(cache->policy, op, false);
1419 		return -EPERM;
1420 	}
1421 
1422 	mg = alloc_migration(cache);
1423 
1424 	mg->op = op;
1425 	mg->overwrite_bio = bio;
1426 
1427 	if (!bio)
1428 		inc_io_migrations(cache);
1429 
1430 	return mg_lock_writes(mg);
1431 }
1432 
1433 /*----------------------------------------------------------------
1434  * invalidation processing
1435  *--------------------------------------------------------------*/
1436 
invalidate_complete(struct dm_cache_migration * mg,bool success)1437 static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1438 {
1439 	struct bio_list bios;
1440 	struct cache *cache = mg->cache;
1441 
1442 	bio_list_init(&bios);
1443 	if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1444 		free_prison_cell(cache, mg->cell);
1445 
1446 	if (!success && mg->overwrite_bio)
1447 		bio_io_error(mg->overwrite_bio);
1448 
1449 	free_migration(mg);
1450 	defer_bios(cache, &bios);
1451 
1452 	background_work_end(cache);
1453 }
1454 
invalidate_completed(struct work_struct * ws)1455 static void invalidate_completed(struct work_struct *ws)
1456 {
1457 	struct dm_cache_migration *mg = ws_to_mg(ws);
1458 	invalidate_complete(mg, !mg->k.input);
1459 }
1460 
invalidate_cblock(struct cache * cache,dm_cblock_t cblock)1461 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1462 {
1463 	int r = policy_invalidate_mapping(cache->policy, cblock);
1464 	if (!r) {
1465 		r = dm_cache_remove_mapping(cache->cmd, cblock);
1466 		if (r) {
1467 			DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1468 				    cache_device_name(cache));
1469 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1470 		}
1471 
1472 	} else if (r == -ENODATA) {
1473 		/*
1474 		 * Harmless, already unmapped.
1475 		 */
1476 		r = 0;
1477 
1478 	} else
1479 		DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1480 
1481 	return r;
1482 }
1483 
invalidate_remove(struct work_struct * ws)1484 static void invalidate_remove(struct work_struct *ws)
1485 {
1486 	int r;
1487 	struct dm_cache_migration *mg = ws_to_mg(ws);
1488 	struct cache *cache = mg->cache;
1489 
1490 	r = invalidate_cblock(cache, mg->invalidate_cblock);
1491 	if (r) {
1492 		invalidate_complete(mg, false);
1493 		return;
1494 	}
1495 
1496 	init_continuation(&mg->k, invalidate_completed);
1497 	continue_after_commit(&cache->committer, &mg->k);
1498 	remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
1499 	mg->overwrite_bio = NULL;
1500 	schedule_commit(&cache->committer);
1501 }
1502 
invalidate_lock(struct dm_cache_migration * mg)1503 static int invalidate_lock(struct dm_cache_migration *mg)
1504 {
1505 	int r;
1506 	struct dm_cell_key_v2 key;
1507 	struct cache *cache = mg->cache;
1508 	struct dm_bio_prison_cell_v2 *prealloc;
1509 
1510 	prealloc = alloc_prison_cell(cache);
1511 
1512 	build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1513 	r = dm_cell_lock_v2(cache->prison, &key,
1514 			    READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1515 	if (r < 0) {
1516 		free_prison_cell(cache, prealloc);
1517 		invalidate_complete(mg, false);
1518 		return r;
1519 	}
1520 
1521 	if (mg->cell != prealloc)
1522 		free_prison_cell(cache, prealloc);
1523 
1524 	if (r)
1525 		quiesce(mg, invalidate_remove);
1526 
1527 	else {
1528 		/*
1529 		 * We can't call invalidate_remove() directly here because we
1530 		 * might still be in request context.
1531 		 */
1532 		init_continuation(&mg->k, invalidate_remove);
1533 		queue_work(cache->wq, &mg->k.ws);
1534 	}
1535 
1536 	return 0;
1537 }
1538 
invalidate_start(struct cache * cache,dm_cblock_t cblock,dm_oblock_t oblock,struct bio * bio)1539 static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1540 			    dm_oblock_t oblock, struct bio *bio)
1541 {
1542 	struct dm_cache_migration *mg;
1543 
1544 	if (!background_work_begin(cache))
1545 		return -EPERM;
1546 
1547 	mg = alloc_migration(cache);
1548 
1549 	mg->overwrite_bio = bio;
1550 	mg->invalidate_cblock = cblock;
1551 	mg->invalidate_oblock = oblock;
1552 
1553 	return invalidate_lock(mg);
1554 }
1555 
1556 /*----------------------------------------------------------------
1557  * bio processing
1558  *--------------------------------------------------------------*/
1559 
1560 enum busy {
1561 	IDLE,
1562 	BUSY
1563 };
1564 
spare_migration_bandwidth(struct cache * cache)1565 static enum busy spare_migration_bandwidth(struct cache *cache)
1566 {
1567 	bool idle = dm_iot_idle_for(&cache->tracker, HZ);
1568 	sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1569 		cache->sectors_per_block;
1570 
1571 	if (idle && current_volume <= cache->migration_threshold)
1572 		return IDLE;
1573 	else
1574 		return BUSY;
1575 }
1576 
inc_hit_counter(struct cache * cache,struct bio * bio)1577 static void inc_hit_counter(struct cache *cache, struct bio *bio)
1578 {
1579 	atomic_inc(bio_data_dir(bio) == READ ?
1580 		   &cache->stats.read_hit : &cache->stats.write_hit);
1581 }
1582 
inc_miss_counter(struct cache * cache,struct bio * bio)1583 static void inc_miss_counter(struct cache *cache, struct bio *bio)
1584 {
1585 	atomic_inc(bio_data_dir(bio) == READ ?
1586 		   &cache->stats.read_miss : &cache->stats.write_miss);
1587 }
1588 
1589 /*----------------------------------------------------------------*/
1590 
map_bio(struct cache * cache,struct bio * bio,dm_oblock_t block,bool * commit_needed)1591 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1592 		   bool *commit_needed)
1593 {
1594 	int r, data_dir;
1595 	bool rb, background_queued;
1596 	dm_cblock_t cblock;
1597 
1598 	*commit_needed = false;
1599 
1600 	rb = bio_detain_shared(cache, block, bio);
1601 	if (!rb) {
1602 		/*
1603 		 * An exclusive lock is held for this block, so we have to
1604 		 * wait.  We set the commit_needed flag so the current
1605 		 * transaction will be committed asap, allowing this lock
1606 		 * to be dropped.
1607 		 */
1608 		*commit_needed = true;
1609 		return DM_MAPIO_SUBMITTED;
1610 	}
1611 
1612 	data_dir = bio_data_dir(bio);
1613 
1614 	if (optimisable_bio(cache, bio, block)) {
1615 		struct policy_work *op = NULL;
1616 
1617 		r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1618 		if (unlikely(r && r != -ENOENT)) {
1619 			DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1620 				    cache_device_name(cache), r);
1621 			bio_io_error(bio);
1622 			return DM_MAPIO_SUBMITTED;
1623 		}
1624 
1625 		if (r == -ENOENT && op) {
1626 			bio_drop_shared_lock(cache, bio);
1627 			BUG_ON(op->op != POLICY_PROMOTE);
1628 			mg_start(cache, op, bio);
1629 			return DM_MAPIO_SUBMITTED;
1630 		}
1631 	} else {
1632 		r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1633 		if (unlikely(r && r != -ENOENT)) {
1634 			DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1635 				    cache_device_name(cache), r);
1636 			bio_io_error(bio);
1637 			return DM_MAPIO_SUBMITTED;
1638 		}
1639 
1640 		if (background_queued)
1641 			wake_migration_worker(cache);
1642 	}
1643 
1644 	if (r == -ENOENT) {
1645 		struct per_bio_data *pb = get_per_bio_data(bio);
1646 
1647 		/*
1648 		 * Miss.
1649 		 */
1650 		inc_miss_counter(cache, bio);
1651 		if (pb->req_nr == 0) {
1652 			accounted_begin(cache, bio);
1653 			remap_to_origin_clear_discard(cache, bio, block);
1654 		} else {
1655 			/*
1656 			 * This is a duplicate writethrough io that is no
1657 			 * longer needed because the block has been demoted.
1658 			 */
1659 			bio_endio(bio);
1660 			return DM_MAPIO_SUBMITTED;
1661 		}
1662 	} else {
1663 		/*
1664 		 * Hit.
1665 		 */
1666 		inc_hit_counter(cache, bio);
1667 
1668 		/*
1669 		 * Passthrough always maps to the origin, invalidating any
1670 		 * cache blocks that are written to.
1671 		 */
1672 		if (passthrough_mode(cache)) {
1673 			if (bio_data_dir(bio) == WRITE) {
1674 				bio_drop_shared_lock(cache, bio);
1675 				atomic_inc(&cache->stats.demotion);
1676 				invalidate_start(cache, cblock, block, bio);
1677 			} else
1678 				remap_to_origin_clear_discard(cache, bio, block);
1679 		} else {
1680 			if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
1681 			    !is_dirty(cache, cblock)) {
1682 				remap_to_origin_and_cache(cache, bio, block, cblock);
1683 				accounted_begin(cache, bio);
1684 			} else
1685 				remap_to_cache_dirty(cache, bio, block, cblock);
1686 		}
1687 	}
1688 
1689 	/*
1690 	 * dm core turns FUA requests into a separate payload and FLUSH req.
1691 	 */
1692 	if (bio->bi_opf & REQ_FUA) {
1693 		/*
1694 		 * issue_after_commit will call accounted_begin a second time.  So
1695 		 * we call accounted_complete() to avoid double accounting.
1696 		 */
1697 		accounted_complete(cache, bio);
1698 		issue_after_commit(&cache->committer, bio);
1699 		*commit_needed = true;
1700 		return DM_MAPIO_SUBMITTED;
1701 	}
1702 
1703 	return DM_MAPIO_REMAPPED;
1704 }
1705 
process_bio(struct cache * cache,struct bio * bio)1706 static bool process_bio(struct cache *cache, struct bio *bio)
1707 {
1708 	bool commit_needed;
1709 
1710 	if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1711 		dm_submit_bio_remap(bio, NULL);
1712 
1713 	return commit_needed;
1714 }
1715 
1716 /*
1717  * A non-zero return indicates read_only or fail_io mode.
1718  */
commit(struct cache * cache,bool clean_shutdown)1719 static int commit(struct cache *cache, bool clean_shutdown)
1720 {
1721 	int r;
1722 
1723 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1724 		return -EINVAL;
1725 
1726 	atomic_inc(&cache->stats.commit_count);
1727 	r = dm_cache_commit(cache->cmd, clean_shutdown);
1728 	if (r)
1729 		metadata_operation_failed(cache, "dm_cache_commit", r);
1730 
1731 	return r;
1732 }
1733 
1734 /*
1735  * Used by the batcher.
1736  */
commit_op(void * context)1737 static blk_status_t commit_op(void *context)
1738 {
1739 	struct cache *cache = context;
1740 
1741 	if (dm_cache_changed_this_transaction(cache->cmd))
1742 		return errno_to_blk_status(commit(cache, false));
1743 
1744 	return 0;
1745 }
1746 
1747 /*----------------------------------------------------------------*/
1748 
process_flush_bio(struct cache * cache,struct bio * bio)1749 static bool process_flush_bio(struct cache *cache, struct bio *bio)
1750 {
1751 	struct per_bio_data *pb = get_per_bio_data(bio);
1752 
1753 	if (!pb->req_nr)
1754 		remap_to_origin(cache, bio);
1755 	else
1756 		remap_to_cache(cache, bio, 0);
1757 
1758 	issue_after_commit(&cache->committer, bio);
1759 	return true;
1760 }
1761 
process_discard_bio(struct cache * cache,struct bio * bio)1762 static bool process_discard_bio(struct cache *cache, struct bio *bio)
1763 {
1764 	dm_dblock_t b, e;
1765 
1766 	// FIXME: do we need to lock the region?  Or can we just assume the
1767 	// user wont be so foolish as to issue discard concurrently with
1768 	// other IO?
1769 	calc_discard_block_range(cache, bio, &b, &e);
1770 	while (b != e) {
1771 		set_discard(cache, b);
1772 		b = to_dblock(from_dblock(b) + 1);
1773 	}
1774 
1775 	if (cache->features.discard_passdown) {
1776 		remap_to_origin(cache, bio);
1777 		dm_submit_bio_remap(bio, NULL);
1778 	} else
1779 		bio_endio(bio);
1780 
1781 	return false;
1782 }
1783 
process_deferred_bios(struct work_struct * ws)1784 static void process_deferred_bios(struct work_struct *ws)
1785 {
1786 	struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1787 
1788 	bool commit_needed = false;
1789 	struct bio_list bios;
1790 	struct bio *bio;
1791 
1792 	bio_list_init(&bios);
1793 
1794 	spin_lock_irq(&cache->lock);
1795 	bio_list_merge(&bios, &cache->deferred_bios);
1796 	bio_list_init(&cache->deferred_bios);
1797 	spin_unlock_irq(&cache->lock);
1798 
1799 	while ((bio = bio_list_pop(&bios))) {
1800 		if (bio->bi_opf & REQ_PREFLUSH)
1801 			commit_needed = process_flush_bio(cache, bio) || commit_needed;
1802 
1803 		else if (bio_op(bio) == REQ_OP_DISCARD)
1804 			commit_needed = process_discard_bio(cache, bio) || commit_needed;
1805 
1806 		else
1807 			commit_needed = process_bio(cache, bio) || commit_needed;
1808 	}
1809 
1810 	if (commit_needed)
1811 		schedule_commit(&cache->committer);
1812 }
1813 
1814 /*----------------------------------------------------------------
1815  * Main worker loop
1816  *--------------------------------------------------------------*/
1817 
requeue_deferred_bios(struct cache * cache)1818 static void requeue_deferred_bios(struct cache *cache)
1819 {
1820 	struct bio *bio;
1821 	struct bio_list bios;
1822 
1823 	bio_list_init(&bios);
1824 	bio_list_merge(&bios, &cache->deferred_bios);
1825 	bio_list_init(&cache->deferred_bios);
1826 
1827 	while ((bio = bio_list_pop(&bios))) {
1828 		bio->bi_status = BLK_STS_DM_REQUEUE;
1829 		bio_endio(bio);
1830 	}
1831 }
1832 
1833 /*
1834  * We want to commit periodically so that not too much
1835  * unwritten metadata builds up.
1836  */
do_waker(struct work_struct * ws)1837 static void do_waker(struct work_struct *ws)
1838 {
1839 	struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1840 
1841 	policy_tick(cache->policy, true);
1842 	wake_migration_worker(cache);
1843 	schedule_commit(&cache->committer);
1844 	queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1845 }
1846 
check_migrations(struct work_struct * ws)1847 static void check_migrations(struct work_struct *ws)
1848 {
1849 	int r;
1850 	struct policy_work *op;
1851 	struct cache *cache = container_of(ws, struct cache, migration_worker);
1852 	enum busy b;
1853 
1854 	for (;;) {
1855 		b = spare_migration_bandwidth(cache);
1856 
1857 		r = policy_get_background_work(cache->policy, b == IDLE, &op);
1858 		if (r == -ENODATA)
1859 			break;
1860 
1861 		if (r) {
1862 			DMERR_LIMIT("%s: policy_background_work failed",
1863 				    cache_device_name(cache));
1864 			break;
1865 		}
1866 
1867 		r = mg_start(cache, op, NULL);
1868 		if (r)
1869 			break;
1870 	}
1871 }
1872 
1873 /*----------------------------------------------------------------
1874  * Target methods
1875  *--------------------------------------------------------------*/
1876 
1877 /*
1878  * This function gets called on the error paths of the constructor, so we
1879  * have to cope with a partially initialised struct.
1880  */
destroy(struct cache * cache)1881 static void destroy(struct cache *cache)
1882 {
1883 	unsigned i;
1884 
1885 	mempool_exit(&cache->migration_pool);
1886 
1887 	if (cache->prison)
1888 		dm_bio_prison_destroy_v2(cache->prison);
1889 
1890 	cancel_delayed_work_sync(&cache->waker);
1891 	if (cache->wq)
1892 		destroy_workqueue(cache->wq);
1893 
1894 	if (cache->dirty_bitset)
1895 		free_bitset(cache->dirty_bitset);
1896 
1897 	if (cache->discard_bitset)
1898 		free_bitset(cache->discard_bitset);
1899 
1900 	if (cache->copier)
1901 		dm_kcopyd_client_destroy(cache->copier);
1902 
1903 	if (cache->cmd)
1904 		dm_cache_metadata_close(cache->cmd);
1905 
1906 	if (cache->metadata_dev)
1907 		dm_put_device(cache->ti, cache->metadata_dev);
1908 
1909 	if (cache->origin_dev)
1910 		dm_put_device(cache->ti, cache->origin_dev);
1911 
1912 	if (cache->cache_dev)
1913 		dm_put_device(cache->ti, cache->cache_dev);
1914 
1915 	if (cache->policy)
1916 		dm_cache_policy_destroy(cache->policy);
1917 
1918 	for (i = 0; i < cache->nr_ctr_args ; i++)
1919 		kfree(cache->ctr_args[i]);
1920 	kfree(cache->ctr_args);
1921 
1922 	bioset_exit(&cache->bs);
1923 
1924 	kfree(cache);
1925 }
1926 
cache_dtr(struct dm_target * ti)1927 static void cache_dtr(struct dm_target *ti)
1928 {
1929 	struct cache *cache = ti->private;
1930 
1931 	destroy(cache);
1932 }
1933 
get_dev_size(struct dm_dev * dev)1934 static sector_t get_dev_size(struct dm_dev *dev)
1935 {
1936 	return bdev_nr_sectors(dev->bdev);
1937 }
1938 
1939 /*----------------------------------------------------------------*/
1940 
1941 /*
1942  * Construct a cache device mapping.
1943  *
1944  * cache <metadata dev> <cache dev> <origin dev> <block size>
1945  *       <#feature args> [<feature arg>]*
1946  *       <policy> <#policy args> [<policy arg>]*
1947  *
1948  * metadata dev    : fast device holding the persistent metadata
1949  * cache dev	   : fast device holding cached data blocks
1950  * origin dev	   : slow device holding original data blocks
1951  * block size	   : cache unit size in sectors
1952  *
1953  * #feature args   : number of feature arguments passed
1954  * feature args    : writethrough.  (The default is writeback.)
1955  *
1956  * policy	   : the replacement policy to use
1957  * #policy args    : an even number of policy arguments corresponding
1958  *		     to key/value pairs passed to the policy
1959  * policy args	   : key/value pairs passed to the policy
1960  *		     E.g. 'sequential_threshold 1024'
1961  *		     See cache-policies.txt for details.
1962  *
1963  * Optional feature arguments are:
1964  *   writethrough  : write through caching that prohibits cache block
1965  *		     content from being different from origin block content.
1966  *		     Without this argument, the default behaviour is to write
1967  *		     back cache block contents later for performance reasons,
1968  *		     so they may differ from the corresponding origin blocks.
1969  */
1970 struct cache_args {
1971 	struct dm_target *ti;
1972 
1973 	struct dm_dev *metadata_dev;
1974 
1975 	struct dm_dev *cache_dev;
1976 	sector_t cache_sectors;
1977 
1978 	struct dm_dev *origin_dev;
1979 	sector_t origin_sectors;
1980 
1981 	uint32_t block_size;
1982 
1983 	const char *policy_name;
1984 	int policy_argc;
1985 	const char **policy_argv;
1986 
1987 	struct cache_features features;
1988 };
1989 
destroy_cache_args(struct cache_args * ca)1990 static void destroy_cache_args(struct cache_args *ca)
1991 {
1992 	if (ca->metadata_dev)
1993 		dm_put_device(ca->ti, ca->metadata_dev);
1994 
1995 	if (ca->cache_dev)
1996 		dm_put_device(ca->ti, ca->cache_dev);
1997 
1998 	if (ca->origin_dev)
1999 		dm_put_device(ca->ti, ca->origin_dev);
2000 
2001 	kfree(ca);
2002 }
2003 
at_least_one_arg(struct dm_arg_set * as,char ** error)2004 static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2005 {
2006 	if (!as->argc) {
2007 		*error = "Insufficient args";
2008 		return false;
2009 	}
2010 
2011 	return true;
2012 }
2013 
parse_metadata_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2014 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2015 			      char **error)
2016 {
2017 	int r;
2018 	sector_t metadata_dev_size;
2019 
2020 	if (!at_least_one_arg(as, error))
2021 		return -EINVAL;
2022 
2023 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2024 			  &ca->metadata_dev);
2025 	if (r) {
2026 		*error = "Error opening metadata device";
2027 		return r;
2028 	}
2029 
2030 	metadata_dev_size = get_dev_size(ca->metadata_dev);
2031 	if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2032 		DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
2033 		       ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS);
2034 
2035 	return 0;
2036 }
2037 
parse_cache_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2038 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2039 			   char **error)
2040 {
2041 	int r;
2042 
2043 	if (!at_least_one_arg(as, error))
2044 		return -EINVAL;
2045 
2046 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2047 			  &ca->cache_dev);
2048 	if (r) {
2049 		*error = "Error opening cache device";
2050 		return r;
2051 	}
2052 	ca->cache_sectors = get_dev_size(ca->cache_dev);
2053 
2054 	return 0;
2055 }
2056 
parse_origin_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2057 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2058 			    char **error)
2059 {
2060 	int r;
2061 
2062 	if (!at_least_one_arg(as, error))
2063 		return -EINVAL;
2064 
2065 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2066 			  &ca->origin_dev);
2067 	if (r) {
2068 		*error = "Error opening origin device";
2069 		return r;
2070 	}
2071 
2072 	ca->origin_sectors = get_dev_size(ca->origin_dev);
2073 	if (ca->ti->len > ca->origin_sectors) {
2074 		*error = "Device size larger than cached device";
2075 		return -EINVAL;
2076 	}
2077 
2078 	return 0;
2079 }
2080 
parse_block_size(struct cache_args * ca,struct dm_arg_set * as,char ** error)2081 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2082 			    char **error)
2083 {
2084 	unsigned long block_size;
2085 
2086 	if (!at_least_one_arg(as, error))
2087 		return -EINVAL;
2088 
2089 	if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
2090 	    block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2091 	    block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
2092 	    block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2093 		*error = "Invalid data block size";
2094 		return -EINVAL;
2095 	}
2096 
2097 	if (block_size > ca->cache_sectors) {
2098 		*error = "Data block size is larger than the cache device";
2099 		return -EINVAL;
2100 	}
2101 
2102 	ca->block_size = block_size;
2103 
2104 	return 0;
2105 }
2106 
init_features(struct cache_features * cf)2107 static void init_features(struct cache_features *cf)
2108 {
2109 	cf->mode = CM_WRITE;
2110 	cf->io_mode = CM_IO_WRITEBACK;
2111 	cf->metadata_version = 1;
2112 	cf->discard_passdown = true;
2113 }
2114 
parse_features(struct cache_args * ca,struct dm_arg_set * as,char ** error)2115 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2116 			  char **error)
2117 {
2118 	static const struct dm_arg _args[] = {
2119 		{0, 3, "Invalid number of cache feature arguments"},
2120 	};
2121 
2122 	int r, mode_ctr = 0;
2123 	unsigned argc;
2124 	const char *arg;
2125 	struct cache_features *cf = &ca->features;
2126 
2127 	init_features(cf);
2128 
2129 	r = dm_read_arg_group(_args, as, &argc, error);
2130 	if (r)
2131 		return -EINVAL;
2132 
2133 	while (argc--) {
2134 		arg = dm_shift_arg(as);
2135 
2136 		if (!strcasecmp(arg, "writeback")) {
2137 			cf->io_mode = CM_IO_WRITEBACK;
2138 			mode_ctr++;
2139 		}
2140 
2141 		else if (!strcasecmp(arg, "writethrough")) {
2142 			cf->io_mode = CM_IO_WRITETHROUGH;
2143 			mode_ctr++;
2144 		}
2145 
2146 		else if (!strcasecmp(arg, "passthrough")) {
2147 			cf->io_mode = CM_IO_PASSTHROUGH;
2148 			mode_ctr++;
2149 		}
2150 
2151 		else if (!strcasecmp(arg, "metadata2"))
2152 			cf->metadata_version = 2;
2153 
2154 		else if (!strcasecmp(arg, "no_discard_passdown"))
2155 			cf->discard_passdown = false;
2156 
2157 		else {
2158 			*error = "Unrecognised cache feature requested";
2159 			return -EINVAL;
2160 		}
2161 	}
2162 
2163 	if (mode_ctr > 1) {
2164 		*error = "Duplicate cache io_mode features requested";
2165 		return -EINVAL;
2166 	}
2167 
2168 	return 0;
2169 }
2170 
parse_policy(struct cache_args * ca,struct dm_arg_set * as,char ** error)2171 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2172 			char **error)
2173 {
2174 	static const struct dm_arg _args[] = {
2175 		{0, 1024, "Invalid number of policy arguments"},
2176 	};
2177 
2178 	int r;
2179 
2180 	if (!at_least_one_arg(as, error))
2181 		return -EINVAL;
2182 
2183 	ca->policy_name = dm_shift_arg(as);
2184 
2185 	r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2186 	if (r)
2187 		return -EINVAL;
2188 
2189 	ca->policy_argv = (const char **)as->argv;
2190 	dm_consume_args(as, ca->policy_argc);
2191 
2192 	return 0;
2193 }
2194 
parse_cache_args(struct cache_args * ca,int argc,char ** argv,char ** error)2195 static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2196 			    char **error)
2197 {
2198 	int r;
2199 	struct dm_arg_set as;
2200 
2201 	as.argc = argc;
2202 	as.argv = argv;
2203 
2204 	r = parse_metadata_dev(ca, &as, error);
2205 	if (r)
2206 		return r;
2207 
2208 	r = parse_cache_dev(ca, &as, error);
2209 	if (r)
2210 		return r;
2211 
2212 	r = parse_origin_dev(ca, &as, error);
2213 	if (r)
2214 		return r;
2215 
2216 	r = parse_block_size(ca, &as, error);
2217 	if (r)
2218 		return r;
2219 
2220 	r = parse_features(ca, &as, error);
2221 	if (r)
2222 		return r;
2223 
2224 	r = parse_policy(ca, &as, error);
2225 	if (r)
2226 		return r;
2227 
2228 	return 0;
2229 }
2230 
2231 /*----------------------------------------------------------------*/
2232 
2233 static struct kmem_cache *migration_cache;
2234 
2235 #define NOT_CORE_OPTION 1
2236 
process_config_option(struct cache * cache,const char * key,const char * value)2237 static int process_config_option(struct cache *cache, const char *key, const char *value)
2238 {
2239 	unsigned long tmp;
2240 
2241 	if (!strcasecmp(key, "migration_threshold")) {
2242 		if (kstrtoul(value, 10, &tmp))
2243 			return -EINVAL;
2244 
2245 		cache->migration_threshold = tmp;
2246 		return 0;
2247 	}
2248 
2249 	return NOT_CORE_OPTION;
2250 }
2251 
set_config_value(struct cache * cache,const char * key,const char * value)2252 static int set_config_value(struct cache *cache, const char *key, const char *value)
2253 {
2254 	int r = process_config_option(cache, key, value);
2255 
2256 	if (r == NOT_CORE_OPTION)
2257 		r = policy_set_config_value(cache->policy, key, value);
2258 
2259 	if (r)
2260 		DMWARN("bad config value for %s: %s", key, value);
2261 
2262 	return r;
2263 }
2264 
set_config_values(struct cache * cache,int argc,const char ** argv)2265 static int set_config_values(struct cache *cache, int argc, const char **argv)
2266 {
2267 	int r = 0;
2268 
2269 	if (argc & 1) {
2270 		DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2271 		return -EINVAL;
2272 	}
2273 
2274 	while (argc) {
2275 		r = set_config_value(cache, argv[0], argv[1]);
2276 		if (r)
2277 			break;
2278 
2279 		argc -= 2;
2280 		argv += 2;
2281 	}
2282 
2283 	return r;
2284 }
2285 
create_cache_policy(struct cache * cache,struct cache_args * ca,char ** error)2286 static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2287 			       char **error)
2288 {
2289 	struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2290 							   cache->cache_size,
2291 							   cache->origin_sectors,
2292 							   cache->sectors_per_block);
2293 	if (IS_ERR(p)) {
2294 		*error = "Error creating cache's policy";
2295 		return PTR_ERR(p);
2296 	}
2297 	cache->policy = p;
2298 	BUG_ON(!cache->policy);
2299 
2300 	return 0;
2301 }
2302 
2303 /*
2304  * We want the discard block size to be at least the size of the cache
2305  * block size and have no more than 2^14 discard blocks across the origin.
2306  */
2307 #define MAX_DISCARD_BLOCKS (1 << 14)
2308 
too_many_discard_blocks(sector_t discard_block_size,sector_t origin_size)2309 static bool too_many_discard_blocks(sector_t discard_block_size,
2310 				    sector_t origin_size)
2311 {
2312 	(void) sector_div(origin_size, discard_block_size);
2313 
2314 	return origin_size > MAX_DISCARD_BLOCKS;
2315 }
2316 
calculate_discard_block_size(sector_t cache_block_size,sector_t origin_size)2317 static sector_t calculate_discard_block_size(sector_t cache_block_size,
2318 					     sector_t origin_size)
2319 {
2320 	sector_t discard_block_size = cache_block_size;
2321 
2322 	if (origin_size)
2323 		while (too_many_discard_blocks(discard_block_size, origin_size))
2324 			discard_block_size *= 2;
2325 
2326 	return discard_block_size;
2327 }
2328 
set_cache_size(struct cache * cache,dm_cblock_t size)2329 static void set_cache_size(struct cache *cache, dm_cblock_t size)
2330 {
2331 	dm_block_t nr_blocks = from_cblock(size);
2332 
2333 	if (nr_blocks > (1 << 20) && cache->cache_size != size)
2334 		DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2335 			     "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2336 			     "Please consider increasing the cache block size to reduce the overall cache block count.",
2337 			     (unsigned long long) nr_blocks);
2338 
2339 	cache->cache_size = size;
2340 }
2341 
2342 #define DEFAULT_MIGRATION_THRESHOLD 2048
2343 
cache_create(struct cache_args * ca,struct cache ** result)2344 static int cache_create(struct cache_args *ca, struct cache **result)
2345 {
2346 	int r = 0;
2347 	char **error = &ca->ti->error;
2348 	struct cache *cache;
2349 	struct dm_target *ti = ca->ti;
2350 	dm_block_t origin_blocks;
2351 	struct dm_cache_metadata *cmd;
2352 	bool may_format = ca->features.mode == CM_WRITE;
2353 
2354 	cache = kzalloc(sizeof(*cache), GFP_KERNEL);
2355 	if (!cache)
2356 		return -ENOMEM;
2357 
2358 	cache->ti = ca->ti;
2359 	ti->private = cache;
2360 	ti->accounts_remapped_io = true;
2361 	ti->num_flush_bios = 2;
2362 	ti->flush_supported = true;
2363 
2364 	ti->num_discard_bios = 1;
2365 	ti->discards_supported = true;
2366 
2367 	ti->per_io_data_size = sizeof(struct per_bio_data);
2368 
2369 	cache->features = ca->features;
2370 	if (writethrough_mode(cache)) {
2371 		/* Create bioset for writethrough bios issued to origin */
2372 		r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
2373 		if (r)
2374 			goto bad;
2375 	}
2376 
2377 	cache->metadata_dev = ca->metadata_dev;
2378 	cache->origin_dev = ca->origin_dev;
2379 	cache->cache_dev = ca->cache_dev;
2380 
2381 	ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2382 
2383 	origin_blocks = cache->origin_sectors = ca->origin_sectors;
2384 	origin_blocks = block_div(origin_blocks, ca->block_size);
2385 	cache->origin_blocks = to_oblock(origin_blocks);
2386 
2387 	cache->sectors_per_block = ca->block_size;
2388 	if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2389 		r = -EINVAL;
2390 		goto bad;
2391 	}
2392 
2393 	if (ca->block_size & (ca->block_size - 1)) {
2394 		dm_block_t cache_size = ca->cache_sectors;
2395 
2396 		cache->sectors_per_block_shift = -1;
2397 		cache_size = block_div(cache_size, ca->block_size);
2398 		set_cache_size(cache, to_cblock(cache_size));
2399 	} else {
2400 		cache->sectors_per_block_shift = __ffs(ca->block_size);
2401 		set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2402 	}
2403 
2404 	r = create_cache_policy(cache, ca, error);
2405 	if (r)
2406 		goto bad;
2407 
2408 	cache->policy_nr_args = ca->policy_argc;
2409 	cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
2410 
2411 	r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
2412 	if (r) {
2413 		*error = "Error setting cache policy's config values";
2414 		goto bad;
2415 	}
2416 
2417 	cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2418 				     ca->block_size, may_format,
2419 				     dm_cache_policy_get_hint_size(cache->policy),
2420 				     ca->features.metadata_version);
2421 	if (IS_ERR(cmd)) {
2422 		*error = "Error creating metadata object";
2423 		r = PTR_ERR(cmd);
2424 		goto bad;
2425 	}
2426 	cache->cmd = cmd;
2427 	set_cache_mode(cache, CM_WRITE);
2428 	if (get_cache_mode(cache) != CM_WRITE) {
2429 		*error = "Unable to get write access to metadata, please check/repair metadata.";
2430 		r = -EINVAL;
2431 		goto bad;
2432 	}
2433 
2434 	if (passthrough_mode(cache)) {
2435 		bool all_clean;
2436 
2437 		r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
2438 		if (r) {
2439 			*error = "dm_cache_metadata_all_clean() failed";
2440 			goto bad;
2441 		}
2442 
2443 		if (!all_clean) {
2444 			*error = "Cannot enter passthrough mode unless all blocks are clean";
2445 			r = -EINVAL;
2446 			goto bad;
2447 		}
2448 
2449 		policy_allow_migrations(cache->policy, false);
2450 	}
2451 
2452 	spin_lock_init(&cache->lock);
2453 	bio_list_init(&cache->deferred_bios);
2454 	atomic_set(&cache->nr_allocated_migrations, 0);
2455 	atomic_set(&cache->nr_io_migrations, 0);
2456 	init_waitqueue_head(&cache->migration_wait);
2457 
2458 	r = -ENOMEM;
2459 	atomic_set(&cache->nr_dirty, 0);
2460 	cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2461 	if (!cache->dirty_bitset) {
2462 		*error = "could not allocate dirty bitset";
2463 		goto bad;
2464 	}
2465 	clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2466 
2467 	cache->discard_block_size =
2468 		calculate_discard_block_size(cache->sectors_per_block,
2469 					     cache->origin_sectors);
2470 	cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
2471 							      cache->discard_block_size));
2472 	cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2473 	if (!cache->discard_bitset) {
2474 		*error = "could not allocate discard bitset";
2475 		goto bad;
2476 	}
2477 	clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2478 
2479 	cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2480 	if (IS_ERR(cache->copier)) {
2481 		*error = "could not create kcopyd client";
2482 		r = PTR_ERR(cache->copier);
2483 		goto bad;
2484 	}
2485 
2486 	cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
2487 	if (!cache->wq) {
2488 		*error = "could not create workqueue for metadata object";
2489 		goto bad;
2490 	}
2491 	INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2492 	INIT_WORK(&cache->migration_worker, check_migrations);
2493 	INIT_DELAYED_WORK(&cache->waker, do_waker);
2494 
2495 	cache->prison = dm_bio_prison_create_v2(cache->wq);
2496 	if (!cache->prison) {
2497 		*error = "could not create bio prison";
2498 		goto bad;
2499 	}
2500 
2501 	r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
2502 				   migration_cache);
2503 	if (r) {
2504 		*error = "Error creating cache's migration mempool";
2505 		goto bad;
2506 	}
2507 
2508 	cache->need_tick_bio = true;
2509 	cache->sized = false;
2510 	cache->invalidate = false;
2511 	cache->commit_requested = false;
2512 	cache->loaded_mappings = false;
2513 	cache->loaded_discards = false;
2514 
2515 	load_stats(cache);
2516 
2517 	atomic_set(&cache->stats.demotion, 0);
2518 	atomic_set(&cache->stats.promotion, 0);
2519 	atomic_set(&cache->stats.copies_avoided, 0);
2520 	atomic_set(&cache->stats.cache_cell_clash, 0);
2521 	atomic_set(&cache->stats.commit_count, 0);
2522 	atomic_set(&cache->stats.discard_count, 0);
2523 
2524 	spin_lock_init(&cache->invalidation_lock);
2525 	INIT_LIST_HEAD(&cache->invalidation_requests);
2526 
2527 	batcher_init(&cache->committer, commit_op, cache,
2528 		     issue_op, cache, cache->wq);
2529 	dm_iot_init(&cache->tracker);
2530 
2531 	init_rwsem(&cache->background_work_lock);
2532 	prevent_background_work(cache);
2533 
2534 	*result = cache;
2535 	return 0;
2536 bad:
2537 	destroy(cache);
2538 	return r;
2539 }
2540 
copy_ctr_args(struct cache * cache,int argc,const char ** argv)2541 static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2542 {
2543 	unsigned i;
2544 	const char **copy;
2545 
2546 	copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2547 	if (!copy)
2548 		return -ENOMEM;
2549 	for (i = 0; i < argc; i++) {
2550 		copy[i] = kstrdup(argv[i], GFP_KERNEL);
2551 		if (!copy[i]) {
2552 			while (i--)
2553 				kfree(copy[i]);
2554 			kfree(copy);
2555 			return -ENOMEM;
2556 		}
2557 	}
2558 
2559 	cache->nr_ctr_args = argc;
2560 	cache->ctr_args = copy;
2561 
2562 	return 0;
2563 }
2564 
cache_ctr(struct dm_target * ti,unsigned argc,char ** argv)2565 static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2566 {
2567 	int r = -EINVAL;
2568 	struct cache_args *ca;
2569 	struct cache *cache = NULL;
2570 
2571 	ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2572 	if (!ca) {
2573 		ti->error = "Error allocating memory for cache";
2574 		return -ENOMEM;
2575 	}
2576 	ca->ti = ti;
2577 
2578 	r = parse_cache_args(ca, argc, argv, &ti->error);
2579 	if (r)
2580 		goto out;
2581 
2582 	r = cache_create(ca, &cache);
2583 	if (r)
2584 		goto out;
2585 
2586 	r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2587 	if (r) {
2588 		destroy(cache);
2589 		goto out;
2590 	}
2591 
2592 	ti->private = cache;
2593 out:
2594 	destroy_cache_args(ca);
2595 	return r;
2596 }
2597 
2598 /*----------------------------------------------------------------*/
2599 
cache_map(struct dm_target * ti,struct bio * bio)2600 static int cache_map(struct dm_target *ti, struct bio *bio)
2601 {
2602 	struct cache *cache = ti->private;
2603 
2604 	int r;
2605 	bool commit_needed;
2606 	dm_oblock_t block = get_bio_block(cache, bio);
2607 
2608 	init_per_bio_data(bio);
2609 	if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2610 		/*
2611 		 * This can only occur if the io goes to a partial block at
2612 		 * the end of the origin device.  We don't cache these.
2613 		 * Just remap to the origin and carry on.
2614 		 */
2615 		remap_to_origin(cache, bio);
2616 		accounted_begin(cache, bio);
2617 		return DM_MAPIO_REMAPPED;
2618 	}
2619 
2620 	if (discard_or_flush(bio)) {
2621 		defer_bio(cache, bio);
2622 		return DM_MAPIO_SUBMITTED;
2623 	}
2624 
2625 	r = map_bio(cache, bio, block, &commit_needed);
2626 	if (commit_needed)
2627 		schedule_commit(&cache->committer);
2628 
2629 	return r;
2630 }
2631 
cache_end_io(struct dm_target * ti,struct bio * bio,blk_status_t * error)2632 static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
2633 {
2634 	struct cache *cache = ti->private;
2635 	unsigned long flags;
2636 	struct per_bio_data *pb = get_per_bio_data(bio);
2637 
2638 	if (pb->tick) {
2639 		policy_tick(cache->policy, false);
2640 
2641 		spin_lock_irqsave(&cache->lock, flags);
2642 		cache->need_tick_bio = true;
2643 		spin_unlock_irqrestore(&cache->lock, flags);
2644 	}
2645 
2646 	bio_drop_shared_lock(cache, bio);
2647 	accounted_complete(cache, bio);
2648 
2649 	return DM_ENDIO_DONE;
2650 }
2651 
write_dirty_bitset(struct cache * cache)2652 static int write_dirty_bitset(struct cache *cache)
2653 {
2654 	int r;
2655 
2656 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2657 		return -EINVAL;
2658 
2659 	r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2660 	if (r)
2661 		metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2662 
2663 	return r;
2664 }
2665 
write_discard_bitset(struct cache * cache)2666 static int write_discard_bitset(struct cache *cache)
2667 {
2668 	unsigned i, r;
2669 
2670 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2671 		return -EINVAL;
2672 
2673 	r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2674 					   cache->discard_nr_blocks);
2675 	if (r) {
2676 		DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2677 		metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2678 		return r;
2679 	}
2680 
2681 	for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2682 		r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2683 					 is_discarded(cache, to_dblock(i)));
2684 		if (r) {
2685 			metadata_operation_failed(cache, "dm_cache_set_discard", r);
2686 			return r;
2687 		}
2688 	}
2689 
2690 	return 0;
2691 }
2692 
write_hints(struct cache * cache)2693 static int write_hints(struct cache *cache)
2694 {
2695 	int r;
2696 
2697 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2698 		return -EINVAL;
2699 
2700 	r = dm_cache_write_hints(cache->cmd, cache->policy);
2701 	if (r) {
2702 		metadata_operation_failed(cache, "dm_cache_write_hints", r);
2703 		return r;
2704 	}
2705 
2706 	return 0;
2707 }
2708 
2709 /*
2710  * returns true on success
2711  */
sync_metadata(struct cache * cache)2712 static bool sync_metadata(struct cache *cache)
2713 {
2714 	int r1, r2, r3, r4;
2715 
2716 	r1 = write_dirty_bitset(cache);
2717 	if (r1)
2718 		DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2719 
2720 	r2 = write_discard_bitset(cache);
2721 	if (r2)
2722 		DMERR("%s: could not write discard bitset", cache_device_name(cache));
2723 
2724 	save_stats(cache);
2725 
2726 	r3 = write_hints(cache);
2727 	if (r3)
2728 		DMERR("%s: could not write hints", cache_device_name(cache));
2729 
2730 	/*
2731 	 * If writing the above metadata failed, we still commit, but don't
2732 	 * set the clean shutdown flag.  This will effectively force every
2733 	 * dirty bit to be set on reload.
2734 	 */
2735 	r4 = commit(cache, !r1 && !r2 && !r3);
2736 	if (r4)
2737 		DMERR("%s: could not write cache metadata", cache_device_name(cache));
2738 
2739 	return !r1 && !r2 && !r3 && !r4;
2740 }
2741 
cache_postsuspend(struct dm_target * ti)2742 static void cache_postsuspend(struct dm_target *ti)
2743 {
2744 	struct cache *cache = ti->private;
2745 
2746 	prevent_background_work(cache);
2747 	BUG_ON(atomic_read(&cache->nr_io_migrations));
2748 
2749 	cancel_delayed_work_sync(&cache->waker);
2750 	drain_workqueue(cache->wq);
2751 	WARN_ON(cache->tracker.in_flight);
2752 
2753 	/*
2754 	 * If it's a flush suspend there won't be any deferred bios, so this
2755 	 * call is harmless.
2756 	 */
2757 	requeue_deferred_bios(cache);
2758 
2759 	if (get_cache_mode(cache) == CM_WRITE)
2760 		(void) sync_metadata(cache);
2761 }
2762 
load_mapping(void * context,dm_oblock_t oblock,dm_cblock_t cblock,bool dirty,uint32_t hint,bool hint_valid)2763 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2764 			bool dirty, uint32_t hint, bool hint_valid)
2765 {
2766 	struct cache *cache = context;
2767 
2768 	if (dirty) {
2769 		set_bit(from_cblock(cblock), cache->dirty_bitset);
2770 		atomic_inc(&cache->nr_dirty);
2771 	} else
2772 		clear_bit(from_cblock(cblock), cache->dirty_bitset);
2773 
2774 	return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
2775 }
2776 
2777 /*
2778  * The discard block size in the on disk metadata is not
2779  * necessarily the same as we're currently using.  So we have to
2780  * be careful to only set the discarded attribute if we know it
2781  * covers a complete block of the new size.
2782  */
2783 struct discard_load_info {
2784 	struct cache *cache;
2785 
2786 	/*
2787 	 * These blocks are sized using the on disk dblock size, rather
2788 	 * than the current one.
2789 	 */
2790 	dm_block_t block_size;
2791 	dm_block_t discard_begin, discard_end;
2792 };
2793 
discard_load_info_init(struct cache * cache,struct discard_load_info * li)2794 static void discard_load_info_init(struct cache *cache,
2795 				   struct discard_load_info *li)
2796 {
2797 	li->cache = cache;
2798 	li->discard_begin = li->discard_end = 0;
2799 }
2800 
set_discard_range(struct discard_load_info * li)2801 static void set_discard_range(struct discard_load_info *li)
2802 {
2803 	sector_t b, e;
2804 
2805 	if (li->discard_begin == li->discard_end)
2806 		return;
2807 
2808 	/*
2809 	 * Convert to sectors.
2810 	 */
2811 	b = li->discard_begin * li->block_size;
2812 	e = li->discard_end * li->block_size;
2813 
2814 	/*
2815 	 * Then convert back to the current dblock size.
2816 	 */
2817 	b = dm_sector_div_up(b, li->cache->discard_block_size);
2818 	sector_div(e, li->cache->discard_block_size);
2819 
2820 	/*
2821 	 * The origin may have shrunk, so we need to check we're still in
2822 	 * bounds.
2823 	 */
2824 	if (e > from_dblock(li->cache->discard_nr_blocks))
2825 		e = from_dblock(li->cache->discard_nr_blocks);
2826 
2827 	for (; b < e; b++)
2828 		set_discard(li->cache, to_dblock(b));
2829 }
2830 
load_discard(void * context,sector_t discard_block_size,dm_dblock_t dblock,bool discard)2831 static int load_discard(void *context, sector_t discard_block_size,
2832 			dm_dblock_t dblock, bool discard)
2833 {
2834 	struct discard_load_info *li = context;
2835 
2836 	li->block_size = discard_block_size;
2837 
2838 	if (discard) {
2839 		if (from_dblock(dblock) == li->discard_end)
2840 			/*
2841 			 * We're already in a discard range, just extend it.
2842 			 */
2843 			li->discard_end = li->discard_end + 1ULL;
2844 
2845 		else {
2846 			/*
2847 			 * Emit the old range and start a new one.
2848 			 */
2849 			set_discard_range(li);
2850 			li->discard_begin = from_dblock(dblock);
2851 			li->discard_end = li->discard_begin + 1ULL;
2852 		}
2853 	} else {
2854 		set_discard_range(li);
2855 		li->discard_begin = li->discard_end = 0;
2856 	}
2857 
2858 	return 0;
2859 }
2860 
get_cache_dev_size(struct cache * cache)2861 static dm_cblock_t get_cache_dev_size(struct cache *cache)
2862 {
2863 	sector_t size = get_dev_size(cache->cache_dev);
2864 	(void) sector_div(size, cache->sectors_per_block);
2865 	return to_cblock(size);
2866 }
2867 
can_resize(struct cache * cache,dm_cblock_t new_size)2868 static bool can_resize(struct cache *cache, dm_cblock_t new_size)
2869 {
2870 	if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
2871 		if (cache->sized) {
2872 			DMERR("%s: unable to extend cache due to missing cache table reload",
2873 			      cache_device_name(cache));
2874 			return false;
2875 		}
2876 	}
2877 
2878 	/*
2879 	 * We can't drop a dirty block when shrinking the cache.
2880 	 */
2881 	while (from_cblock(new_size) < from_cblock(cache->cache_size)) {
2882 		new_size = to_cblock(from_cblock(new_size) + 1);
2883 		if (is_dirty(cache, new_size)) {
2884 			DMERR("%s: unable to shrink cache; cache block %llu is dirty",
2885 			      cache_device_name(cache),
2886 			      (unsigned long long) from_cblock(new_size));
2887 			return false;
2888 		}
2889 	}
2890 
2891 	return true;
2892 }
2893 
resize_cache_dev(struct cache * cache,dm_cblock_t new_size)2894 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
2895 {
2896 	int r;
2897 
2898 	r = dm_cache_resize(cache->cmd, new_size);
2899 	if (r) {
2900 		DMERR("%s: could not resize cache metadata", cache_device_name(cache));
2901 		metadata_operation_failed(cache, "dm_cache_resize", r);
2902 		return r;
2903 	}
2904 
2905 	set_cache_size(cache, new_size);
2906 
2907 	return 0;
2908 }
2909 
cache_preresume(struct dm_target * ti)2910 static int cache_preresume(struct dm_target *ti)
2911 {
2912 	int r = 0;
2913 	struct cache *cache = ti->private;
2914 	dm_cblock_t csize = get_cache_dev_size(cache);
2915 
2916 	/*
2917 	 * Check to see if the cache has resized.
2918 	 */
2919 	if (!cache->sized) {
2920 		r = resize_cache_dev(cache, csize);
2921 		if (r)
2922 			return r;
2923 
2924 		cache->sized = true;
2925 
2926 	} else if (csize != cache->cache_size) {
2927 		if (!can_resize(cache, csize))
2928 			return -EINVAL;
2929 
2930 		r = resize_cache_dev(cache, csize);
2931 		if (r)
2932 			return r;
2933 	}
2934 
2935 	if (!cache->loaded_mappings) {
2936 		r = dm_cache_load_mappings(cache->cmd, cache->policy,
2937 					   load_mapping, cache);
2938 		if (r) {
2939 			DMERR("%s: could not load cache mappings", cache_device_name(cache));
2940 			metadata_operation_failed(cache, "dm_cache_load_mappings", r);
2941 			return r;
2942 		}
2943 
2944 		cache->loaded_mappings = true;
2945 	}
2946 
2947 	if (!cache->loaded_discards) {
2948 		struct discard_load_info li;
2949 
2950 		/*
2951 		 * The discard bitset could have been resized, or the
2952 		 * discard block size changed.  To be safe we start by
2953 		 * setting every dblock to not discarded.
2954 		 */
2955 		clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2956 
2957 		discard_load_info_init(cache, &li);
2958 		r = dm_cache_load_discards(cache->cmd, load_discard, &li);
2959 		if (r) {
2960 			DMERR("%s: could not load origin discards", cache_device_name(cache));
2961 			metadata_operation_failed(cache, "dm_cache_load_discards", r);
2962 			return r;
2963 		}
2964 		set_discard_range(&li);
2965 
2966 		cache->loaded_discards = true;
2967 	}
2968 
2969 	return r;
2970 }
2971 
cache_resume(struct dm_target * ti)2972 static void cache_resume(struct dm_target *ti)
2973 {
2974 	struct cache *cache = ti->private;
2975 
2976 	cache->need_tick_bio = true;
2977 	allow_background_work(cache);
2978 	do_waker(&cache->waker.work);
2979 }
2980 
emit_flags(struct cache * cache,char * result,unsigned maxlen,ssize_t * sz_ptr)2981 static void emit_flags(struct cache *cache, char *result,
2982 		       unsigned maxlen, ssize_t *sz_ptr)
2983 {
2984 	ssize_t sz = *sz_ptr;
2985 	struct cache_features *cf = &cache->features;
2986 	unsigned count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
2987 
2988 	DMEMIT("%u ", count);
2989 
2990 	if (cf->metadata_version == 2)
2991 		DMEMIT("metadata2 ");
2992 
2993 	if (writethrough_mode(cache))
2994 		DMEMIT("writethrough ");
2995 
2996 	else if (passthrough_mode(cache))
2997 		DMEMIT("passthrough ");
2998 
2999 	else if (writeback_mode(cache))
3000 		DMEMIT("writeback ");
3001 
3002 	else {
3003 		DMEMIT("unknown ");
3004 		DMERR("%s: internal error: unknown io mode: %d",
3005 		      cache_device_name(cache), (int) cf->io_mode);
3006 	}
3007 
3008 	if (!cf->discard_passdown)
3009 		DMEMIT("no_discard_passdown ");
3010 
3011 	*sz_ptr = sz;
3012 }
3013 
3014 /*
3015  * Status format:
3016  *
3017  * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3018  * <cache block size> <#used cache blocks>/<#total cache blocks>
3019  * <#read hits> <#read misses> <#write hits> <#write misses>
3020  * <#demotions> <#promotions> <#dirty>
3021  * <#features> <features>*
3022  * <#core args> <core args>
3023  * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3024  */
cache_status(struct dm_target * ti,status_type_t type,unsigned status_flags,char * result,unsigned maxlen)3025 static void cache_status(struct dm_target *ti, status_type_t type,
3026 			 unsigned status_flags, char *result, unsigned maxlen)
3027 {
3028 	int r = 0;
3029 	unsigned i;
3030 	ssize_t sz = 0;
3031 	dm_block_t nr_free_blocks_metadata = 0;
3032 	dm_block_t nr_blocks_metadata = 0;
3033 	char buf[BDEVNAME_SIZE];
3034 	struct cache *cache = ti->private;
3035 	dm_cblock_t residency;
3036 	bool needs_check;
3037 
3038 	switch (type) {
3039 	case STATUSTYPE_INFO:
3040 		if (get_cache_mode(cache) == CM_FAIL) {
3041 			DMEMIT("Fail");
3042 			break;
3043 		}
3044 
3045 		/* Commit to ensure statistics aren't out-of-date */
3046 		if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3047 			(void) commit(cache, false);
3048 
3049 		r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3050 		if (r) {
3051 			DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3052 			      cache_device_name(cache), r);
3053 			goto err;
3054 		}
3055 
3056 		r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3057 		if (r) {
3058 			DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3059 			      cache_device_name(cache), r);
3060 			goto err;
3061 		}
3062 
3063 		residency = policy_residency(cache->policy);
3064 
3065 		DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
3066 		       (unsigned)DM_CACHE_METADATA_BLOCK_SIZE,
3067 		       (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3068 		       (unsigned long long)nr_blocks_metadata,
3069 		       (unsigned long long)cache->sectors_per_block,
3070 		       (unsigned long long) from_cblock(residency),
3071 		       (unsigned long long) from_cblock(cache->cache_size),
3072 		       (unsigned) atomic_read(&cache->stats.read_hit),
3073 		       (unsigned) atomic_read(&cache->stats.read_miss),
3074 		       (unsigned) atomic_read(&cache->stats.write_hit),
3075 		       (unsigned) atomic_read(&cache->stats.write_miss),
3076 		       (unsigned) atomic_read(&cache->stats.demotion),
3077 		       (unsigned) atomic_read(&cache->stats.promotion),
3078 		       (unsigned long) atomic_read(&cache->nr_dirty));
3079 
3080 		emit_flags(cache, result, maxlen, &sz);
3081 
3082 		DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
3083 
3084 		DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3085 		if (sz < maxlen) {
3086 			r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3087 			if (r)
3088 				DMERR("%s: policy_emit_config_values returned %d",
3089 				      cache_device_name(cache), r);
3090 		}
3091 
3092 		if (get_cache_mode(cache) == CM_READ_ONLY)
3093 			DMEMIT("ro ");
3094 		else
3095 			DMEMIT("rw ");
3096 
3097 		r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3098 
3099 		if (r || needs_check)
3100 			DMEMIT("needs_check ");
3101 		else
3102 			DMEMIT("- ");
3103 
3104 		break;
3105 
3106 	case STATUSTYPE_TABLE:
3107 		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3108 		DMEMIT("%s ", buf);
3109 		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3110 		DMEMIT("%s ", buf);
3111 		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3112 		DMEMIT("%s", buf);
3113 
3114 		for (i = 0; i < cache->nr_ctr_args - 1; i++)
3115 			DMEMIT(" %s", cache->ctr_args[i]);
3116 		if (cache->nr_ctr_args)
3117 			DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
3118 		break;
3119 
3120 	case STATUSTYPE_IMA:
3121 		DMEMIT_TARGET_NAME_VERSION(ti->type);
3122 		if (get_cache_mode(cache) == CM_FAIL)
3123 			DMEMIT(",metadata_mode=fail");
3124 		else if (get_cache_mode(cache) == CM_READ_ONLY)
3125 			DMEMIT(",metadata_mode=ro");
3126 		else
3127 			DMEMIT(",metadata_mode=rw");
3128 
3129 		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3130 		DMEMIT(",cache_metadata_device=%s", buf);
3131 		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3132 		DMEMIT(",cache_device=%s", buf);
3133 		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3134 		DMEMIT(",cache_origin_device=%s", buf);
3135 		DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n');
3136 		DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n');
3137 		DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n');
3138 		DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n');
3139 		DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y');
3140 		DMEMIT(";");
3141 		break;
3142 	}
3143 
3144 	return;
3145 
3146 err:
3147 	DMEMIT("Error");
3148 }
3149 
3150 /*
3151  * Defines a range of cblocks, begin to (end - 1) are in the range.  end is
3152  * the one-past-the-end value.
3153  */
3154 struct cblock_range {
3155 	dm_cblock_t begin;
3156 	dm_cblock_t end;
3157 };
3158 
3159 /*
3160  * A cache block range can take two forms:
3161  *
3162  * i) A single cblock, eg. '3456'
3163  * ii) A begin and end cblock with a dash between, eg. 123-234
3164  */
parse_cblock_range(struct cache * cache,const char * str,struct cblock_range * result)3165 static int parse_cblock_range(struct cache *cache, const char *str,
3166 			      struct cblock_range *result)
3167 {
3168 	char dummy;
3169 	uint64_t b, e;
3170 	int r;
3171 
3172 	/*
3173 	 * Try and parse form (ii) first.
3174 	 */
3175 	r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
3176 	if (r < 0)
3177 		return r;
3178 
3179 	if (r == 2) {
3180 		result->begin = to_cblock(b);
3181 		result->end = to_cblock(e);
3182 		return 0;
3183 	}
3184 
3185 	/*
3186 	 * That didn't work, try form (i).
3187 	 */
3188 	r = sscanf(str, "%llu%c", &b, &dummy);
3189 	if (r < 0)
3190 		return r;
3191 
3192 	if (r == 1) {
3193 		result->begin = to_cblock(b);
3194 		result->end = to_cblock(from_cblock(result->begin) + 1u);
3195 		return 0;
3196 	}
3197 
3198 	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
3199 	return -EINVAL;
3200 }
3201 
validate_cblock_range(struct cache * cache,struct cblock_range * range)3202 static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
3203 {
3204 	uint64_t b = from_cblock(range->begin);
3205 	uint64_t e = from_cblock(range->end);
3206 	uint64_t n = from_cblock(cache->cache_size);
3207 
3208 	if (b >= n) {
3209 		DMERR("%s: begin cblock out of range: %llu >= %llu",
3210 		      cache_device_name(cache), b, n);
3211 		return -EINVAL;
3212 	}
3213 
3214 	if (e > n) {
3215 		DMERR("%s: end cblock out of range: %llu > %llu",
3216 		      cache_device_name(cache), e, n);
3217 		return -EINVAL;
3218 	}
3219 
3220 	if (b >= e) {
3221 		DMERR("%s: invalid cblock range: %llu >= %llu",
3222 		      cache_device_name(cache), b, e);
3223 		return -EINVAL;
3224 	}
3225 
3226 	return 0;
3227 }
3228 
cblock_succ(dm_cblock_t b)3229 static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3230 {
3231 	return to_cblock(from_cblock(b) + 1);
3232 }
3233 
request_invalidation(struct cache * cache,struct cblock_range * range)3234 static int request_invalidation(struct cache *cache, struct cblock_range *range)
3235 {
3236 	int r = 0;
3237 
3238 	/*
3239 	 * We don't need to do any locking here because we know we're in
3240 	 * passthrough mode.  There's is potential for a race between an
3241 	 * invalidation triggered by an io and an invalidation message.  This
3242 	 * is harmless, we must not worry if the policy call fails.
3243 	 */
3244 	while (range->begin != range->end) {
3245 		r = invalidate_cblock(cache, range->begin);
3246 		if (r)
3247 			return r;
3248 
3249 		range->begin = cblock_succ(range->begin);
3250 	}
3251 
3252 	cache->commit_requested = true;
3253 	return r;
3254 }
3255 
process_invalidate_cblocks_message(struct cache * cache,unsigned count,const char ** cblock_ranges)3256 static int process_invalidate_cblocks_message(struct cache *cache, unsigned count,
3257 					      const char **cblock_ranges)
3258 {
3259 	int r = 0;
3260 	unsigned i;
3261 	struct cblock_range range;
3262 
3263 	if (!passthrough_mode(cache)) {
3264 		DMERR("%s: cache has to be in passthrough mode for invalidation",
3265 		      cache_device_name(cache));
3266 		return -EPERM;
3267 	}
3268 
3269 	for (i = 0; i < count; i++) {
3270 		r = parse_cblock_range(cache, cblock_ranges[i], &range);
3271 		if (r)
3272 			break;
3273 
3274 		r = validate_cblock_range(cache, &range);
3275 		if (r)
3276 			break;
3277 
3278 		/*
3279 		 * Pass begin and end origin blocks to the worker and wake it.
3280 		 */
3281 		r = request_invalidation(cache, &range);
3282 		if (r)
3283 			break;
3284 	}
3285 
3286 	return r;
3287 }
3288 
3289 /*
3290  * Supports
3291  *	"<key> <value>"
3292  * and
3293  *     "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3294  *
3295  * The key migration_threshold is supported by the cache target core.
3296  */
cache_message(struct dm_target * ti,unsigned argc,char ** argv,char * result,unsigned maxlen)3297 static int cache_message(struct dm_target *ti, unsigned argc, char **argv,
3298 			 char *result, unsigned maxlen)
3299 {
3300 	struct cache *cache = ti->private;
3301 
3302 	if (!argc)
3303 		return -EINVAL;
3304 
3305 	if (get_cache_mode(cache) >= CM_READ_ONLY) {
3306 		DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3307 		      cache_device_name(cache));
3308 		return -EOPNOTSUPP;
3309 	}
3310 
3311 	if (!strcasecmp(argv[0], "invalidate_cblocks"))
3312 		return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
3313 
3314 	if (argc != 2)
3315 		return -EINVAL;
3316 
3317 	return set_config_value(cache, argv[0], argv[1]);
3318 }
3319 
cache_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)3320 static int cache_iterate_devices(struct dm_target *ti,
3321 				 iterate_devices_callout_fn fn, void *data)
3322 {
3323 	int r = 0;
3324 	struct cache *cache = ti->private;
3325 
3326 	r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3327 	if (!r)
3328 		r = fn(ti, cache->origin_dev, 0, ti->len, data);
3329 
3330 	return r;
3331 }
3332 
3333 /*
3334  * If discard_passdown was enabled verify that the origin device
3335  * supports discards.  Disable discard_passdown if not.
3336  */
disable_passdown_if_not_supported(struct cache * cache)3337 static void disable_passdown_if_not_supported(struct cache *cache)
3338 {
3339 	struct block_device *origin_bdev = cache->origin_dev->bdev;
3340 	struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3341 	const char *reason = NULL;
3342 
3343 	if (!cache->features.discard_passdown)
3344 		return;
3345 
3346 	if (!bdev_max_discard_sectors(origin_bdev))
3347 		reason = "discard unsupported";
3348 
3349 	else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
3350 		reason = "max discard sectors smaller than a block";
3351 
3352 	if (reason) {
3353 		DMWARN("Origin device (%pg) %s: Disabling discard passdown.",
3354 		       origin_bdev, reason);
3355 		cache->features.discard_passdown = false;
3356 	}
3357 }
3358 
set_discard_limits(struct cache * cache,struct queue_limits * limits)3359 static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3360 {
3361 	struct block_device *origin_bdev = cache->origin_dev->bdev;
3362 	struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3363 
3364 	if (!cache->features.discard_passdown) {
3365 		/* No passdown is done so setting own virtual limits */
3366 		limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3367 						    cache->origin_sectors);
3368 		limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3369 		return;
3370 	}
3371 
3372 	/*
3373 	 * cache_iterate_devices() is stacking both origin and fast device limits
3374 	 * but discards aren't passed to fast device, so inherit origin's limits.
3375 	 */
3376 	limits->max_discard_sectors = origin_limits->max_discard_sectors;
3377 	limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
3378 	limits->discard_granularity = origin_limits->discard_granularity;
3379 	limits->discard_alignment = origin_limits->discard_alignment;
3380 	limits->discard_misaligned = origin_limits->discard_misaligned;
3381 }
3382 
cache_io_hints(struct dm_target * ti,struct queue_limits * limits)3383 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3384 {
3385 	struct cache *cache = ti->private;
3386 	uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3387 
3388 	/*
3389 	 * If the system-determined stacked limits are compatible with the
3390 	 * cache's blocksize (io_opt is a factor) do not override them.
3391 	 */
3392 	if (io_opt_sectors < cache->sectors_per_block ||
3393 	    do_div(io_opt_sectors, cache->sectors_per_block)) {
3394 		blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
3395 		blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
3396 	}
3397 
3398 	disable_passdown_if_not_supported(cache);
3399 	set_discard_limits(cache, limits);
3400 }
3401 
3402 /*----------------------------------------------------------------*/
3403 
3404 static struct target_type cache_target = {
3405 	.name = "cache",
3406 	.version = {2, 2, 0},
3407 	.module = THIS_MODULE,
3408 	.ctr = cache_ctr,
3409 	.dtr = cache_dtr,
3410 	.map = cache_map,
3411 	.end_io = cache_end_io,
3412 	.postsuspend = cache_postsuspend,
3413 	.preresume = cache_preresume,
3414 	.resume = cache_resume,
3415 	.status = cache_status,
3416 	.message = cache_message,
3417 	.iterate_devices = cache_iterate_devices,
3418 	.io_hints = cache_io_hints,
3419 };
3420 
dm_cache_init(void)3421 static int __init dm_cache_init(void)
3422 {
3423 	int r;
3424 
3425 	migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3426 	if (!migration_cache)
3427 		return -ENOMEM;
3428 
3429 	r = dm_register_target(&cache_target);
3430 	if (r) {
3431 		DMERR("cache target registration failed: %d", r);
3432 		kmem_cache_destroy(migration_cache);
3433 		return r;
3434 	}
3435 
3436 	return 0;
3437 }
3438 
dm_cache_exit(void)3439 static void __exit dm_cache_exit(void)
3440 {
3441 	dm_unregister_target(&cache_target);
3442 	kmem_cache_destroy(migration_cache);
3443 }
3444 
3445 module_init(dm_cache_init);
3446 module_exit(dm_cache_exit);
3447 
3448 MODULE_DESCRIPTION(DM_NAME " cache target");
3449 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
3450 MODULE_LICENSE("GPL");
3451