1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * raid10.c : Multiple Devices driver for Linux
4  *
5  * Copyright (C) 2000-2004 Neil Brown
6  *
7  * RAID-10 support for md.
8  *
9  * Base on code in raid1.c.  See raid1.c for further copyright information.
10  */
11 
12 #include <linux/slab.h>
13 #include <linux/delay.h>
14 #include <linux/blkdev.h>
15 #include <linux/module.h>
16 #include <linux/seq_file.h>
17 #include <linux/ratelimit.h>
18 #include <linux/kthread.h>
19 #include <linux/raid/md_p.h>
20 #include <trace/events/block.h>
21 #include "md.h"
22 #include "raid10.h"
23 #include "raid0.h"
24 #include "md-bitmap.h"
25 
26 /*
27  * RAID10 provides a combination of RAID0 and RAID1 functionality.
28  * The layout of data is defined by
29  *    chunk_size
30  *    raid_disks
31  *    near_copies (stored in low byte of layout)
32  *    far_copies (stored in second byte of layout)
33  *    far_offset (stored in bit 16 of layout )
34  *    use_far_sets (stored in bit 17 of layout )
35  *    use_far_sets_bugfixed (stored in bit 18 of layout )
36  *
37  * The data to be stored is divided into chunks using chunksize.  Each device
38  * is divided into far_copies sections.   In each section, chunks are laid out
39  * in a style similar to raid0, but near_copies copies of each chunk is stored
40  * (each on a different drive).  The starting device for each section is offset
41  * near_copies from the starting device of the previous section.  Thus there
42  * are (near_copies * far_copies) of each chunk, and each is on a different
43  * drive.  near_copies and far_copies must be at least one, and their product
44  * is at most raid_disks.
45  *
46  * If far_offset is true, then the far_copies are handled a bit differently.
47  * The copies are still in different stripes, but instead of being very far
48  * apart on disk, there are adjacent stripes.
49  *
50  * The far and offset algorithms are handled slightly differently if
51  * 'use_far_sets' is true.  In this case, the array's devices are grouped into
52  * sets that are (near_copies * far_copies) in size.  The far copied stripes
53  * are still shifted by 'near_copies' devices, but this shifting stays confined
54  * to the set rather than the entire array.  This is done to improve the number
55  * of device combinations that can fail without causing the array to fail.
56  * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
57  * on a device):
58  *    A B C D    A B C D E
59  *      ...         ...
60  *    D A B C    E A B C D
61  * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
62  *    [A B] [C D]    [A B] [C D E]
63  *    |...| |...|    |...| | ... |
64  *    [B A] [D C]    [B A] [E C D]
65  */
66 
67 static void allow_barrier(struct r10conf *conf);
68 static void lower_barrier(struct r10conf *conf);
69 static int _enough(struct r10conf *conf, int previous, int ignore);
70 static int enough(struct r10conf *conf, int ignore);
71 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
72 				int *skipped);
73 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
74 static void end_reshape_write(struct bio *bio);
75 static void end_reshape(struct r10conf *conf);
76 
77 #define raid10_log(md, fmt, args...)				\
78 	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
79 
80 #include "raid1-10.c"
81 
82 #define NULL_CMD
83 #define cmd_before(conf, cmd) \
84 	do { \
85 		write_sequnlock_irq(&(conf)->resync_lock); \
86 		cmd; \
87 	} while (0)
88 #define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)
89 
90 #define wait_event_barrier_cmd(conf, cond, cmd) \
91 	wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
92 		       cmd_after(conf))
93 
94 #define wait_event_barrier(conf, cond) \
95 	wait_event_barrier_cmd(conf, cond, NULL_CMD)
96 
97 /*
98  * for resync bio, r10bio pointer can be retrieved from the per-bio
99  * 'struct resync_pages'.
100  */
get_resync_r10bio(struct bio * bio)101 static inline struct r10bio *get_resync_r10bio(struct bio *bio)
102 {
103 	return get_resync_pages(bio)->raid_bio;
104 }
105 
r10bio_pool_alloc(gfp_t gfp_flags,void * data)106 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
107 {
108 	struct r10conf *conf = data;
109 	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);
110 
111 	/* allocate a r10bio with room for raid_disks entries in the
112 	 * bios array */
113 	return kzalloc(size, gfp_flags);
114 }
115 
116 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
117 /* amount of memory to reserve for resync requests */
118 #define RESYNC_WINDOW (1024*1024)
119 /* maximum number of concurrent requests, memory permitting */
120 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
121 #define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
122 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
123 
124 /*
125  * When performing a resync, we need to read and compare, so
126  * we need as many pages are there are copies.
127  * When performing a recovery, we need 2 bios, one for read,
128  * one for write (we recover only one drive per r10buf)
129  *
130  */
r10buf_pool_alloc(gfp_t gfp_flags,void * data)131 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
132 {
133 	struct r10conf *conf = data;
134 	struct r10bio *r10_bio;
135 	struct bio *bio;
136 	int j;
137 	int nalloc, nalloc_rp;
138 	struct resync_pages *rps;
139 
140 	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
141 	if (!r10_bio)
142 		return NULL;
143 
144 	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
145 	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
146 		nalloc = conf->copies; /* resync */
147 	else
148 		nalloc = 2; /* recovery */
149 
150 	/* allocate once for all bios */
151 	if (!conf->have_replacement)
152 		nalloc_rp = nalloc;
153 	else
154 		nalloc_rp = nalloc * 2;
155 	rps = kmalloc_array(nalloc_rp, sizeof(struct resync_pages), gfp_flags);
156 	if (!rps)
157 		goto out_free_r10bio;
158 
159 	/*
160 	 * Allocate bios.
161 	 */
162 	for (j = nalloc ; j-- ; ) {
163 		bio = bio_kmalloc(RESYNC_PAGES, gfp_flags);
164 		if (!bio)
165 			goto out_free_bio;
166 		bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);
167 		r10_bio->devs[j].bio = bio;
168 		if (!conf->have_replacement)
169 			continue;
170 		bio = bio_kmalloc(RESYNC_PAGES, gfp_flags);
171 		if (!bio)
172 			goto out_free_bio;
173 		bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);
174 		r10_bio->devs[j].repl_bio = bio;
175 	}
176 	/*
177 	 * Allocate RESYNC_PAGES data pages and attach them
178 	 * where needed.
179 	 */
180 	for (j = 0; j < nalloc; j++) {
181 		struct bio *rbio = r10_bio->devs[j].repl_bio;
182 		struct resync_pages *rp, *rp_repl;
183 
184 		rp = &rps[j];
185 		if (rbio)
186 			rp_repl = &rps[nalloc + j];
187 
188 		bio = r10_bio->devs[j].bio;
189 
190 		if (!j || test_bit(MD_RECOVERY_SYNC,
191 				   &conf->mddev->recovery)) {
192 			if (resync_alloc_pages(rp, gfp_flags))
193 				goto out_free_pages;
194 		} else {
195 			memcpy(rp, &rps[0], sizeof(*rp));
196 			resync_get_all_pages(rp);
197 		}
198 
199 		rp->raid_bio = r10_bio;
200 		bio->bi_private = rp;
201 		if (rbio) {
202 			memcpy(rp_repl, rp, sizeof(*rp));
203 			rbio->bi_private = rp_repl;
204 		}
205 	}
206 
207 	return r10_bio;
208 
209 out_free_pages:
210 	while (--j >= 0)
211 		resync_free_pages(&rps[j]);
212 
213 	j = 0;
214 out_free_bio:
215 	for ( ; j < nalloc; j++) {
216 		if (r10_bio->devs[j].bio)
217 			bio_uninit(r10_bio->devs[j].bio);
218 		kfree(r10_bio->devs[j].bio);
219 		if (r10_bio->devs[j].repl_bio)
220 			bio_uninit(r10_bio->devs[j].repl_bio);
221 		kfree(r10_bio->devs[j].repl_bio);
222 	}
223 	kfree(rps);
224 out_free_r10bio:
225 	rbio_pool_free(r10_bio, conf);
226 	return NULL;
227 }
228 
r10buf_pool_free(void * __r10_bio,void * data)229 static void r10buf_pool_free(void *__r10_bio, void *data)
230 {
231 	struct r10conf *conf = data;
232 	struct r10bio *r10bio = __r10_bio;
233 	int j;
234 	struct resync_pages *rp = NULL;
235 
236 	for (j = conf->copies; j--; ) {
237 		struct bio *bio = r10bio->devs[j].bio;
238 
239 		if (bio) {
240 			rp = get_resync_pages(bio);
241 			resync_free_pages(rp);
242 			bio_uninit(bio);
243 			kfree(bio);
244 		}
245 
246 		bio = r10bio->devs[j].repl_bio;
247 		if (bio) {
248 			bio_uninit(bio);
249 			kfree(bio);
250 		}
251 	}
252 
253 	/* resync pages array stored in the 1st bio's .bi_private */
254 	kfree(rp);
255 
256 	rbio_pool_free(r10bio, conf);
257 }
258 
put_all_bios(struct r10conf * conf,struct r10bio * r10_bio)259 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
260 {
261 	int i;
262 
263 	for (i = 0; i < conf->geo.raid_disks; i++) {
264 		struct bio **bio = & r10_bio->devs[i].bio;
265 		if (!BIO_SPECIAL(*bio))
266 			bio_put(*bio);
267 		*bio = NULL;
268 		bio = &r10_bio->devs[i].repl_bio;
269 		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
270 			bio_put(*bio);
271 		*bio = NULL;
272 	}
273 }
274 
free_r10bio(struct r10bio * r10_bio)275 static void free_r10bio(struct r10bio *r10_bio)
276 {
277 	struct r10conf *conf = r10_bio->mddev->private;
278 
279 	put_all_bios(conf, r10_bio);
280 	mempool_free(r10_bio, &conf->r10bio_pool);
281 }
282 
put_buf(struct r10bio * r10_bio)283 static void put_buf(struct r10bio *r10_bio)
284 {
285 	struct r10conf *conf = r10_bio->mddev->private;
286 
287 	mempool_free(r10_bio, &conf->r10buf_pool);
288 
289 	lower_barrier(conf);
290 }
291 
wake_up_barrier(struct r10conf * conf)292 static void wake_up_barrier(struct r10conf *conf)
293 {
294 	if (wq_has_sleeper(&conf->wait_barrier))
295 		wake_up(&conf->wait_barrier);
296 }
297 
reschedule_retry(struct r10bio * r10_bio)298 static void reschedule_retry(struct r10bio *r10_bio)
299 {
300 	unsigned long flags;
301 	struct mddev *mddev = r10_bio->mddev;
302 	struct r10conf *conf = mddev->private;
303 
304 	spin_lock_irqsave(&conf->device_lock, flags);
305 	list_add(&r10_bio->retry_list, &conf->retry_list);
306 	conf->nr_queued ++;
307 	spin_unlock_irqrestore(&conf->device_lock, flags);
308 
309 	/* wake up frozen array... */
310 	wake_up(&conf->wait_barrier);
311 
312 	md_wakeup_thread(mddev->thread);
313 }
314 
315 /*
316  * raid_end_bio_io() is called when we have finished servicing a mirrored
317  * operation and are ready to return a success/failure code to the buffer
318  * cache layer.
319  */
raid_end_bio_io(struct r10bio * r10_bio)320 static void raid_end_bio_io(struct r10bio *r10_bio)
321 {
322 	struct bio *bio = r10_bio->master_bio;
323 	struct r10conf *conf = r10_bio->mddev->private;
324 
325 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
326 		bio->bi_status = BLK_STS_IOERR;
327 
328 	if (blk_queue_io_stat(bio->bi_bdev->bd_disk->queue))
329 		bio_end_io_acct(bio, r10_bio->start_time);
330 	bio_endio(bio);
331 	/*
332 	 * Wake up any possible resync thread that waits for the device
333 	 * to go idle.
334 	 */
335 	allow_barrier(conf);
336 
337 	free_r10bio(r10_bio);
338 }
339 
340 /*
341  * Update disk head position estimator based on IRQ completion info.
342  */
update_head_pos(int slot,struct r10bio * r10_bio)343 static inline void update_head_pos(int slot, struct r10bio *r10_bio)
344 {
345 	struct r10conf *conf = r10_bio->mddev->private;
346 
347 	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
348 		r10_bio->devs[slot].addr + (r10_bio->sectors);
349 }
350 
351 /*
352  * Find the disk number which triggered given bio
353  */
find_bio_disk(struct r10conf * conf,struct r10bio * r10_bio,struct bio * bio,int * slotp,int * replp)354 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
355 			 struct bio *bio, int *slotp, int *replp)
356 {
357 	int slot;
358 	int repl = 0;
359 
360 	for (slot = 0; slot < conf->geo.raid_disks; slot++) {
361 		if (r10_bio->devs[slot].bio == bio)
362 			break;
363 		if (r10_bio->devs[slot].repl_bio == bio) {
364 			repl = 1;
365 			break;
366 		}
367 	}
368 
369 	update_head_pos(slot, r10_bio);
370 
371 	if (slotp)
372 		*slotp = slot;
373 	if (replp)
374 		*replp = repl;
375 	return r10_bio->devs[slot].devnum;
376 }
377 
raid10_end_read_request(struct bio * bio)378 static void raid10_end_read_request(struct bio *bio)
379 {
380 	int uptodate = !bio->bi_status;
381 	struct r10bio *r10_bio = bio->bi_private;
382 	int slot;
383 	struct md_rdev *rdev;
384 	struct r10conf *conf = r10_bio->mddev->private;
385 
386 	slot = r10_bio->read_slot;
387 	rdev = r10_bio->devs[slot].rdev;
388 	/*
389 	 * this branch is our 'one mirror IO has finished' event handler:
390 	 */
391 	update_head_pos(slot, r10_bio);
392 
393 	if (uptodate) {
394 		/*
395 		 * Set R10BIO_Uptodate in our master bio, so that
396 		 * we will return a good error code to the higher
397 		 * levels even if IO on some other mirrored buffer fails.
398 		 *
399 		 * The 'master' represents the composite IO operation to
400 		 * user-side. So if something waits for IO, then it will
401 		 * wait for the 'master' bio.
402 		 */
403 		set_bit(R10BIO_Uptodate, &r10_bio->state);
404 	} else {
405 		/* If all other devices that store this block have
406 		 * failed, we want to return the error upwards rather
407 		 * than fail the last device.  Here we redefine
408 		 * "uptodate" to mean "Don't want to retry"
409 		 */
410 		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
411 			     rdev->raid_disk))
412 			uptodate = 1;
413 	}
414 	if (uptodate) {
415 		raid_end_bio_io(r10_bio);
416 		rdev_dec_pending(rdev, conf->mddev);
417 	} else {
418 		/*
419 		 * oops, read error - keep the refcount on the rdev
420 		 */
421 		pr_err_ratelimited("md/raid10:%s: %pg: rescheduling sector %llu\n",
422 				   mdname(conf->mddev),
423 				   rdev->bdev,
424 				   (unsigned long long)r10_bio->sector);
425 		set_bit(R10BIO_ReadError, &r10_bio->state);
426 		reschedule_retry(r10_bio);
427 	}
428 }
429 
close_write(struct r10bio * r10_bio)430 static void close_write(struct r10bio *r10_bio)
431 {
432 	/* clear the bitmap if all writes complete successfully */
433 	md_bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
434 			   r10_bio->sectors,
435 			   !test_bit(R10BIO_Degraded, &r10_bio->state),
436 			   0);
437 	md_write_end(r10_bio->mddev);
438 }
439 
one_write_done(struct r10bio * r10_bio)440 static void one_write_done(struct r10bio *r10_bio)
441 {
442 	if (atomic_dec_and_test(&r10_bio->remaining)) {
443 		if (test_bit(R10BIO_WriteError, &r10_bio->state))
444 			reschedule_retry(r10_bio);
445 		else {
446 			close_write(r10_bio);
447 			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
448 				reschedule_retry(r10_bio);
449 			else
450 				raid_end_bio_io(r10_bio);
451 		}
452 	}
453 }
454 
raid10_end_write_request(struct bio * bio)455 static void raid10_end_write_request(struct bio *bio)
456 {
457 	struct r10bio *r10_bio = bio->bi_private;
458 	int dev;
459 	int dec_rdev = 1;
460 	struct r10conf *conf = r10_bio->mddev->private;
461 	int slot, repl;
462 	struct md_rdev *rdev = NULL;
463 	struct bio *to_put = NULL;
464 	bool discard_error;
465 
466 	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
467 
468 	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
469 
470 	if (repl)
471 		rdev = conf->mirrors[dev].replacement;
472 	if (!rdev) {
473 		smp_rmb();
474 		repl = 0;
475 		rdev = conf->mirrors[dev].rdev;
476 	}
477 	/*
478 	 * this branch is our 'one mirror IO has finished' event handler:
479 	 */
480 	if (bio->bi_status && !discard_error) {
481 		if (repl)
482 			/* Never record new bad blocks to replacement,
483 			 * just fail it.
484 			 */
485 			md_error(rdev->mddev, rdev);
486 		else {
487 			set_bit(WriteErrorSeen,	&rdev->flags);
488 			if (!test_and_set_bit(WantReplacement, &rdev->flags))
489 				set_bit(MD_RECOVERY_NEEDED,
490 					&rdev->mddev->recovery);
491 
492 			dec_rdev = 0;
493 			if (test_bit(FailFast, &rdev->flags) &&
494 			    (bio->bi_opf & MD_FAILFAST)) {
495 				md_error(rdev->mddev, rdev);
496 			}
497 
498 			/*
499 			 * When the device is faulty, it is not necessary to
500 			 * handle write error.
501 			 */
502 			if (!test_bit(Faulty, &rdev->flags))
503 				set_bit(R10BIO_WriteError, &r10_bio->state);
504 			else {
505 				/* Fail the request */
506 				set_bit(R10BIO_Degraded, &r10_bio->state);
507 				r10_bio->devs[slot].bio = NULL;
508 				to_put = bio;
509 				dec_rdev = 1;
510 			}
511 		}
512 	} else {
513 		/*
514 		 * Set R10BIO_Uptodate in our master bio, so that
515 		 * we will return a good error code for to the higher
516 		 * levels even if IO on some other mirrored buffer fails.
517 		 *
518 		 * The 'master' represents the composite IO operation to
519 		 * user-side. So if something waits for IO, then it will
520 		 * wait for the 'master' bio.
521 		 */
522 		sector_t first_bad;
523 		int bad_sectors;
524 
525 		/*
526 		 * Do not set R10BIO_Uptodate if the current device is
527 		 * rebuilding or Faulty. This is because we cannot use
528 		 * such device for properly reading the data back (we could
529 		 * potentially use it, if the current write would have felt
530 		 * before rdev->recovery_offset, but for simplicity we don't
531 		 * check this here.
532 		 */
533 		if (test_bit(In_sync, &rdev->flags) &&
534 		    !test_bit(Faulty, &rdev->flags))
535 			set_bit(R10BIO_Uptodate, &r10_bio->state);
536 
537 		/* Maybe we can clear some bad blocks. */
538 		if (is_badblock(rdev,
539 				r10_bio->devs[slot].addr,
540 				r10_bio->sectors,
541 				&first_bad, &bad_sectors) && !discard_error) {
542 			bio_put(bio);
543 			if (repl)
544 				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
545 			else
546 				r10_bio->devs[slot].bio = IO_MADE_GOOD;
547 			dec_rdev = 0;
548 			set_bit(R10BIO_MadeGood, &r10_bio->state);
549 		}
550 	}
551 
552 	/*
553 	 *
554 	 * Let's see if all mirrored write operations have finished
555 	 * already.
556 	 */
557 	one_write_done(r10_bio);
558 	if (dec_rdev)
559 		rdev_dec_pending(rdev, conf->mddev);
560 	if (to_put)
561 		bio_put(to_put);
562 }
563 
564 /*
565  * RAID10 layout manager
566  * As well as the chunksize and raid_disks count, there are two
567  * parameters: near_copies and far_copies.
568  * near_copies * far_copies must be <= raid_disks.
569  * Normally one of these will be 1.
570  * If both are 1, we get raid0.
571  * If near_copies == raid_disks, we get raid1.
572  *
573  * Chunks are laid out in raid0 style with near_copies copies of the
574  * first chunk, followed by near_copies copies of the next chunk and
575  * so on.
576  * If far_copies > 1, then after 1/far_copies of the array has been assigned
577  * as described above, we start again with a device offset of near_copies.
578  * So we effectively have another copy of the whole array further down all
579  * the drives, but with blocks on different drives.
580  * With this layout, and block is never stored twice on the one device.
581  *
582  * raid10_find_phys finds the sector offset of a given virtual sector
583  * on each device that it is on.
584  *
585  * raid10_find_virt does the reverse mapping, from a device and a
586  * sector offset to a virtual address
587  */
588 
__raid10_find_phys(struct geom * geo,struct r10bio * r10bio)589 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
590 {
591 	int n,f;
592 	sector_t sector;
593 	sector_t chunk;
594 	sector_t stripe;
595 	int dev;
596 	int slot = 0;
597 	int last_far_set_start, last_far_set_size;
598 
599 	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
600 	last_far_set_start *= geo->far_set_size;
601 
602 	last_far_set_size = geo->far_set_size;
603 	last_far_set_size += (geo->raid_disks % geo->far_set_size);
604 
605 	/* now calculate first sector/dev */
606 	chunk = r10bio->sector >> geo->chunk_shift;
607 	sector = r10bio->sector & geo->chunk_mask;
608 
609 	chunk *= geo->near_copies;
610 	stripe = chunk;
611 	dev = sector_div(stripe, geo->raid_disks);
612 	if (geo->far_offset)
613 		stripe *= geo->far_copies;
614 
615 	sector += stripe << geo->chunk_shift;
616 
617 	/* and calculate all the others */
618 	for (n = 0; n < geo->near_copies; n++) {
619 		int d = dev;
620 		int set;
621 		sector_t s = sector;
622 		r10bio->devs[slot].devnum = d;
623 		r10bio->devs[slot].addr = s;
624 		slot++;
625 
626 		for (f = 1; f < geo->far_copies; f++) {
627 			set = d / geo->far_set_size;
628 			d += geo->near_copies;
629 
630 			if ((geo->raid_disks % geo->far_set_size) &&
631 			    (d > last_far_set_start)) {
632 				d -= last_far_set_start;
633 				d %= last_far_set_size;
634 				d += last_far_set_start;
635 			} else {
636 				d %= geo->far_set_size;
637 				d += geo->far_set_size * set;
638 			}
639 			s += geo->stride;
640 			r10bio->devs[slot].devnum = d;
641 			r10bio->devs[slot].addr = s;
642 			slot++;
643 		}
644 		dev++;
645 		if (dev >= geo->raid_disks) {
646 			dev = 0;
647 			sector += (geo->chunk_mask + 1);
648 		}
649 	}
650 }
651 
raid10_find_phys(struct r10conf * conf,struct r10bio * r10bio)652 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
653 {
654 	struct geom *geo = &conf->geo;
655 
656 	if (conf->reshape_progress != MaxSector &&
657 	    ((r10bio->sector >= conf->reshape_progress) !=
658 	     conf->mddev->reshape_backwards)) {
659 		set_bit(R10BIO_Previous, &r10bio->state);
660 		geo = &conf->prev;
661 	} else
662 		clear_bit(R10BIO_Previous, &r10bio->state);
663 
664 	__raid10_find_phys(geo, r10bio);
665 }
666 
raid10_find_virt(struct r10conf * conf,sector_t sector,int dev)667 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
668 {
669 	sector_t offset, chunk, vchunk;
670 	/* Never use conf->prev as this is only called during resync
671 	 * or recovery, so reshape isn't happening
672 	 */
673 	struct geom *geo = &conf->geo;
674 	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
675 	int far_set_size = geo->far_set_size;
676 	int last_far_set_start;
677 
678 	if (geo->raid_disks % geo->far_set_size) {
679 		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
680 		last_far_set_start *= geo->far_set_size;
681 
682 		if (dev >= last_far_set_start) {
683 			far_set_size = geo->far_set_size;
684 			far_set_size += (geo->raid_disks % geo->far_set_size);
685 			far_set_start = last_far_set_start;
686 		}
687 	}
688 
689 	offset = sector & geo->chunk_mask;
690 	if (geo->far_offset) {
691 		int fc;
692 		chunk = sector >> geo->chunk_shift;
693 		fc = sector_div(chunk, geo->far_copies);
694 		dev -= fc * geo->near_copies;
695 		if (dev < far_set_start)
696 			dev += far_set_size;
697 	} else {
698 		while (sector >= geo->stride) {
699 			sector -= geo->stride;
700 			if (dev < (geo->near_copies + far_set_start))
701 				dev += far_set_size - geo->near_copies;
702 			else
703 				dev -= geo->near_copies;
704 		}
705 		chunk = sector >> geo->chunk_shift;
706 	}
707 	vchunk = chunk * geo->raid_disks + dev;
708 	sector_div(vchunk, geo->near_copies);
709 	return (vchunk << geo->chunk_shift) + offset;
710 }
711 
712 /*
713  * This routine returns the disk from which the requested read should
714  * be done. There is a per-array 'next expected sequential IO' sector
715  * number - if this matches on the next IO then we use the last disk.
716  * There is also a per-disk 'last know head position' sector that is
717  * maintained from IRQ contexts, both the normal and the resync IO
718  * completion handlers update this position correctly. If there is no
719  * perfect sequential match then we pick the disk whose head is closest.
720  *
721  * If there are 2 mirrors in the same 2 devices, performance degrades
722  * because position is mirror, not device based.
723  *
724  * The rdev for the device selected will have nr_pending incremented.
725  */
726 
727 /*
728  * FIXME: possibly should rethink readbalancing and do it differently
729  * depending on near_copies / far_copies geometry.
730  */
read_balance(struct r10conf * conf,struct r10bio * r10_bio,int * max_sectors)731 static struct md_rdev *read_balance(struct r10conf *conf,
732 				    struct r10bio *r10_bio,
733 				    int *max_sectors)
734 {
735 	const sector_t this_sector = r10_bio->sector;
736 	int disk, slot;
737 	int sectors = r10_bio->sectors;
738 	int best_good_sectors;
739 	sector_t new_distance, best_dist;
740 	struct md_rdev *best_dist_rdev, *best_pending_rdev, *rdev = NULL;
741 	int do_balance;
742 	int best_dist_slot, best_pending_slot;
743 	bool has_nonrot_disk = false;
744 	unsigned int min_pending;
745 	struct geom *geo = &conf->geo;
746 
747 	raid10_find_phys(conf, r10_bio);
748 	rcu_read_lock();
749 	best_dist_slot = -1;
750 	min_pending = UINT_MAX;
751 	best_dist_rdev = NULL;
752 	best_pending_rdev = NULL;
753 	best_dist = MaxSector;
754 	best_good_sectors = 0;
755 	do_balance = 1;
756 	clear_bit(R10BIO_FailFast, &r10_bio->state);
757 	/*
758 	 * Check if we can balance. We can balance on the whole
759 	 * device if no resync is going on (recovery is ok), or below
760 	 * the resync window. We take the first readable disk when
761 	 * above the resync window.
762 	 */
763 	if ((conf->mddev->recovery_cp < MaxSector
764 	     && (this_sector + sectors >= conf->next_resync)) ||
765 	    (mddev_is_clustered(conf->mddev) &&
766 	     md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,
767 					    this_sector + sectors)))
768 		do_balance = 0;
769 
770 	for (slot = 0; slot < conf->copies ; slot++) {
771 		sector_t first_bad;
772 		int bad_sectors;
773 		sector_t dev_sector;
774 		unsigned int pending;
775 		bool nonrot;
776 
777 		if (r10_bio->devs[slot].bio == IO_BLOCKED)
778 			continue;
779 		disk = r10_bio->devs[slot].devnum;
780 		rdev = rcu_dereference(conf->mirrors[disk].replacement);
781 		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
782 		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
783 			rdev = rcu_dereference(conf->mirrors[disk].rdev);
784 		if (rdev == NULL ||
785 		    test_bit(Faulty, &rdev->flags))
786 			continue;
787 		if (!test_bit(In_sync, &rdev->flags) &&
788 		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
789 			continue;
790 
791 		dev_sector = r10_bio->devs[slot].addr;
792 		if (is_badblock(rdev, dev_sector, sectors,
793 				&first_bad, &bad_sectors)) {
794 			if (best_dist < MaxSector)
795 				/* Already have a better slot */
796 				continue;
797 			if (first_bad <= dev_sector) {
798 				/* Cannot read here.  If this is the
799 				 * 'primary' device, then we must not read
800 				 * beyond 'bad_sectors' from another device.
801 				 */
802 				bad_sectors -= (dev_sector - first_bad);
803 				if (!do_balance && sectors > bad_sectors)
804 					sectors = bad_sectors;
805 				if (best_good_sectors > sectors)
806 					best_good_sectors = sectors;
807 			} else {
808 				sector_t good_sectors =
809 					first_bad - dev_sector;
810 				if (good_sectors > best_good_sectors) {
811 					best_good_sectors = good_sectors;
812 					best_dist_slot = slot;
813 					best_dist_rdev = rdev;
814 				}
815 				if (!do_balance)
816 					/* Must read from here */
817 					break;
818 			}
819 			continue;
820 		} else
821 			best_good_sectors = sectors;
822 
823 		if (!do_balance)
824 			break;
825 
826 		nonrot = bdev_nonrot(rdev->bdev);
827 		has_nonrot_disk |= nonrot;
828 		pending = atomic_read(&rdev->nr_pending);
829 		if (min_pending > pending && nonrot) {
830 			min_pending = pending;
831 			best_pending_slot = slot;
832 			best_pending_rdev = rdev;
833 		}
834 
835 		if (best_dist_slot >= 0)
836 			/* At least 2 disks to choose from so failfast is OK */
837 			set_bit(R10BIO_FailFast, &r10_bio->state);
838 		/* This optimisation is debatable, and completely destroys
839 		 * sequential read speed for 'far copies' arrays.  So only
840 		 * keep it for 'near' arrays, and review those later.
841 		 */
842 		if (geo->near_copies > 1 && !pending)
843 			new_distance = 0;
844 
845 		/* for far > 1 always use the lowest address */
846 		else if (geo->far_copies > 1)
847 			new_distance = r10_bio->devs[slot].addr;
848 		else
849 			new_distance = abs(r10_bio->devs[slot].addr -
850 					   conf->mirrors[disk].head_position);
851 
852 		if (new_distance < best_dist) {
853 			best_dist = new_distance;
854 			best_dist_slot = slot;
855 			best_dist_rdev = rdev;
856 		}
857 	}
858 	if (slot >= conf->copies) {
859 		if (has_nonrot_disk) {
860 			slot = best_pending_slot;
861 			rdev = best_pending_rdev;
862 		} else {
863 			slot = best_dist_slot;
864 			rdev = best_dist_rdev;
865 		}
866 	}
867 
868 	if (slot >= 0) {
869 		atomic_inc(&rdev->nr_pending);
870 		r10_bio->read_slot = slot;
871 	} else
872 		rdev = NULL;
873 	rcu_read_unlock();
874 	*max_sectors = best_good_sectors;
875 
876 	return rdev;
877 }
878 
flush_pending_writes(struct r10conf * conf)879 static void flush_pending_writes(struct r10conf *conf)
880 {
881 	/* Any writes that have been queued but are awaiting
882 	 * bitmap updates get flushed here.
883 	 */
884 	spin_lock_irq(&conf->device_lock);
885 
886 	if (conf->pending_bio_list.head) {
887 		struct blk_plug plug;
888 		struct bio *bio;
889 
890 		bio = bio_list_get(&conf->pending_bio_list);
891 		spin_unlock_irq(&conf->device_lock);
892 
893 		/*
894 		 * As this is called in a wait_event() loop (see freeze_array),
895 		 * current->state might be TASK_UNINTERRUPTIBLE which will
896 		 * cause a warning when we prepare to wait again.  As it is
897 		 * rare that this path is taken, it is perfectly safe to force
898 		 * us to go around the wait_event() loop again, so the warning
899 		 * is a false-positive. Silence the warning by resetting
900 		 * thread state
901 		 */
902 		__set_current_state(TASK_RUNNING);
903 
904 		blk_start_plug(&plug);
905 		/* flush any pending bitmap writes to disk
906 		 * before proceeding w/ I/O */
907 		md_bitmap_unplug(conf->mddev->bitmap);
908 		wake_up(&conf->wait_barrier);
909 
910 		while (bio) { /* submit pending writes */
911 			struct bio *next = bio->bi_next;
912 			struct md_rdev *rdev = (void*)bio->bi_bdev;
913 			bio->bi_next = NULL;
914 			bio_set_dev(bio, rdev->bdev);
915 			if (test_bit(Faulty, &rdev->flags)) {
916 				bio_io_error(bio);
917 			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
918 					    !bdev_max_discard_sectors(bio->bi_bdev)))
919 				/* Just ignore it */
920 				bio_endio(bio);
921 			else
922 				submit_bio_noacct(bio);
923 			bio = next;
924 		}
925 		blk_finish_plug(&plug);
926 	} else
927 		spin_unlock_irq(&conf->device_lock);
928 }
929 
930 /* Barriers....
931  * Sometimes we need to suspend IO while we do something else,
932  * either some resync/recovery, or reconfigure the array.
933  * To do this we raise a 'barrier'.
934  * The 'barrier' is a counter that can be raised multiple times
935  * to count how many activities are happening which preclude
936  * normal IO.
937  * We can only raise the barrier if there is no pending IO.
938  * i.e. if nr_pending == 0.
939  * We choose only to raise the barrier if no-one is waiting for the
940  * barrier to go down.  This means that as soon as an IO request
941  * is ready, no other operations which require a barrier will start
942  * until the IO request has had a chance.
943  *
944  * So: regular IO calls 'wait_barrier'.  When that returns there
945  *    is no backgroup IO happening,  It must arrange to call
946  *    allow_barrier when it has finished its IO.
947  * backgroup IO calls must call raise_barrier.  Once that returns
948  *    there is no normal IO happeing.  It must arrange to call
949  *    lower_barrier when the particular background IO completes.
950  */
951 
raise_barrier(struct r10conf * conf,int force)952 static void raise_barrier(struct r10conf *conf, int force)
953 {
954 	write_seqlock_irq(&conf->resync_lock);
955 	BUG_ON(force && !conf->barrier);
956 
957 	/* Wait until no block IO is waiting (unless 'force') */
958 	wait_event_barrier(conf, force || !conf->nr_waiting);
959 
960 	/* block any new IO from starting */
961 	WRITE_ONCE(conf->barrier, conf->barrier + 1);
962 
963 	/* Now wait for all pending IO to complete */
964 	wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&
965 				 conf->barrier < RESYNC_DEPTH);
966 
967 	write_sequnlock_irq(&conf->resync_lock);
968 }
969 
lower_barrier(struct r10conf * conf)970 static void lower_barrier(struct r10conf *conf)
971 {
972 	unsigned long flags;
973 
974 	write_seqlock_irqsave(&conf->resync_lock, flags);
975 	WRITE_ONCE(conf->barrier, conf->barrier - 1);
976 	write_sequnlock_irqrestore(&conf->resync_lock, flags);
977 	wake_up(&conf->wait_barrier);
978 }
979 
stop_waiting_barrier(struct r10conf * conf)980 static bool stop_waiting_barrier(struct r10conf *conf)
981 {
982 	struct bio_list *bio_list = current->bio_list;
983 
984 	/* barrier is dropped */
985 	if (!conf->barrier)
986 		return true;
987 
988 	/*
989 	 * If there are already pending requests (preventing the barrier from
990 	 * rising completely), and the pre-process bio queue isn't empty, then
991 	 * don't wait, as we need to empty that queue to get the nr_pending
992 	 * count down.
993 	 */
994 	if (atomic_read(&conf->nr_pending) && bio_list &&
995 	    (!bio_list_empty(&bio_list[0]) || !bio_list_empty(&bio_list[1])))
996 		return true;
997 
998 	/* move on if recovery thread is blocked by us */
999 	if (conf->mddev->thread->tsk == current &&
1000 	    test_bit(MD_RECOVERY_RUNNING, &conf->mddev->recovery) &&
1001 	    conf->nr_queued > 0)
1002 		return true;
1003 
1004 	return false;
1005 }
1006 
wait_barrier_nolock(struct r10conf * conf)1007 static bool wait_barrier_nolock(struct r10conf *conf)
1008 {
1009 	unsigned int seq = read_seqbegin(&conf->resync_lock);
1010 
1011 	if (READ_ONCE(conf->barrier))
1012 		return false;
1013 
1014 	atomic_inc(&conf->nr_pending);
1015 	if (!read_seqretry(&conf->resync_lock, seq))
1016 		return true;
1017 
1018 	if (atomic_dec_and_test(&conf->nr_pending))
1019 		wake_up_barrier(conf);
1020 
1021 	return false;
1022 }
1023 
wait_barrier(struct r10conf * conf,bool nowait)1024 static bool wait_barrier(struct r10conf *conf, bool nowait)
1025 {
1026 	bool ret = true;
1027 
1028 	if (wait_barrier_nolock(conf))
1029 		return true;
1030 
1031 	write_seqlock_irq(&conf->resync_lock);
1032 	if (conf->barrier) {
1033 		/* Return false when nowait flag is set */
1034 		if (nowait) {
1035 			ret = false;
1036 		} else {
1037 			conf->nr_waiting++;
1038 			raid10_log(conf->mddev, "wait barrier");
1039 			wait_event_barrier(conf, stop_waiting_barrier(conf));
1040 			conf->nr_waiting--;
1041 		}
1042 		if (!conf->nr_waiting)
1043 			wake_up(&conf->wait_barrier);
1044 	}
1045 	/* Only increment nr_pending when we wait */
1046 	if (ret)
1047 		atomic_inc(&conf->nr_pending);
1048 	write_sequnlock_irq(&conf->resync_lock);
1049 	return ret;
1050 }
1051 
allow_barrier(struct r10conf * conf)1052 static void allow_barrier(struct r10conf *conf)
1053 {
1054 	if ((atomic_dec_and_test(&conf->nr_pending)) ||
1055 			(conf->array_freeze_pending))
1056 		wake_up_barrier(conf);
1057 }
1058 
freeze_array(struct r10conf * conf,int extra)1059 static void freeze_array(struct r10conf *conf, int extra)
1060 {
1061 	/* stop syncio and normal IO and wait for everything to
1062 	 * go quiet.
1063 	 * We increment barrier and nr_waiting, and then
1064 	 * wait until nr_pending match nr_queued+extra
1065 	 * This is called in the context of one normal IO request
1066 	 * that has failed. Thus any sync request that might be pending
1067 	 * will be blocked by nr_pending, and we need to wait for
1068 	 * pending IO requests to complete or be queued for re-try.
1069 	 * Thus the number queued (nr_queued) plus this request (extra)
1070 	 * must match the number of pending IOs (nr_pending) before
1071 	 * we continue.
1072 	 */
1073 	write_seqlock_irq(&conf->resync_lock);
1074 	conf->array_freeze_pending++;
1075 	WRITE_ONCE(conf->barrier, conf->barrier + 1);
1076 	conf->nr_waiting++;
1077 	wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
1078 			conf->nr_queued + extra, flush_pending_writes(conf));
1079 	conf->array_freeze_pending--;
1080 	write_sequnlock_irq(&conf->resync_lock);
1081 }
1082 
unfreeze_array(struct r10conf * conf)1083 static void unfreeze_array(struct r10conf *conf)
1084 {
1085 	/* reverse the effect of the freeze */
1086 	write_seqlock_irq(&conf->resync_lock);
1087 	WRITE_ONCE(conf->barrier, conf->barrier - 1);
1088 	conf->nr_waiting--;
1089 	wake_up(&conf->wait_barrier);
1090 	write_sequnlock_irq(&conf->resync_lock);
1091 }
1092 
choose_data_offset(struct r10bio * r10_bio,struct md_rdev * rdev)1093 static sector_t choose_data_offset(struct r10bio *r10_bio,
1094 				   struct md_rdev *rdev)
1095 {
1096 	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1097 	    test_bit(R10BIO_Previous, &r10_bio->state))
1098 		return rdev->data_offset;
1099 	else
1100 		return rdev->new_data_offset;
1101 }
1102 
raid10_unplug(struct blk_plug_cb * cb,bool from_schedule)1103 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1104 {
1105 	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, cb);
1106 	struct mddev *mddev = plug->cb.data;
1107 	struct r10conf *conf = mddev->private;
1108 	struct bio *bio;
1109 
1110 	if (from_schedule || current->bio_list) {
1111 		spin_lock_irq(&conf->device_lock);
1112 		bio_list_merge(&conf->pending_bio_list, &plug->pending);
1113 		spin_unlock_irq(&conf->device_lock);
1114 		wake_up(&conf->wait_barrier);
1115 		md_wakeup_thread(mddev->thread);
1116 		kfree(plug);
1117 		return;
1118 	}
1119 
1120 	/* we aren't scheduling, so we can do the write-out directly. */
1121 	bio = bio_list_get(&plug->pending);
1122 	md_bitmap_unplug(mddev->bitmap);
1123 	wake_up(&conf->wait_barrier);
1124 
1125 	while (bio) { /* submit pending writes */
1126 		struct bio *next = bio->bi_next;
1127 		struct md_rdev *rdev = (void*)bio->bi_bdev;
1128 		bio->bi_next = NULL;
1129 		bio_set_dev(bio, rdev->bdev);
1130 		if (test_bit(Faulty, &rdev->flags)) {
1131 			bio_io_error(bio);
1132 		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
1133 				    !bdev_max_discard_sectors(bio->bi_bdev)))
1134 			/* Just ignore it */
1135 			bio_endio(bio);
1136 		else
1137 			submit_bio_noacct(bio);
1138 		bio = next;
1139 	}
1140 	kfree(plug);
1141 }
1142 
1143 /*
1144  * 1. Register the new request and wait if the reconstruction thread has put
1145  * up a bar for new requests. Continue immediately if no resync is active
1146  * currently.
1147  * 2. If IO spans the reshape position.  Need to wait for reshape to pass.
1148  */
regular_request_wait(struct mddev * mddev,struct r10conf * conf,struct bio * bio,sector_t sectors)1149 static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
1150 				 struct bio *bio, sector_t sectors)
1151 {
1152 	/* Bail out if REQ_NOWAIT is set for the bio */
1153 	if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {
1154 		bio_wouldblock_error(bio);
1155 		return false;
1156 	}
1157 	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1158 	    bio->bi_iter.bi_sector < conf->reshape_progress &&
1159 	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1160 		allow_barrier(conf);
1161 		if (bio->bi_opf & REQ_NOWAIT) {
1162 			bio_wouldblock_error(bio);
1163 			return false;
1164 		}
1165 		raid10_log(conf->mddev, "wait reshape");
1166 		wait_event(conf->wait_barrier,
1167 			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
1168 			   conf->reshape_progress >= bio->bi_iter.bi_sector +
1169 			   sectors);
1170 		wait_barrier(conf, false);
1171 	}
1172 	return true;
1173 }
1174 
raid10_read_request(struct mddev * mddev,struct bio * bio,struct r10bio * r10_bio)1175 static void raid10_read_request(struct mddev *mddev, struct bio *bio,
1176 				struct r10bio *r10_bio)
1177 {
1178 	struct r10conf *conf = mddev->private;
1179 	struct bio *read_bio;
1180 	const enum req_op op = bio_op(bio);
1181 	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1182 	int max_sectors;
1183 	struct md_rdev *rdev;
1184 	char b[BDEVNAME_SIZE];
1185 	int slot = r10_bio->read_slot;
1186 	struct md_rdev *err_rdev = NULL;
1187 	gfp_t gfp = GFP_NOIO;
1188 
1189 	if (slot >= 0 && r10_bio->devs[slot].rdev) {
1190 		/*
1191 		 * This is an error retry, but we cannot
1192 		 * safely dereference the rdev in the r10_bio,
1193 		 * we must use the one in conf.
1194 		 * If it has already been disconnected (unlikely)
1195 		 * we lose the device name in error messages.
1196 		 */
1197 		int disk;
1198 		/*
1199 		 * As we are blocking raid10, it is a little safer to
1200 		 * use __GFP_HIGH.
1201 		 */
1202 		gfp = GFP_NOIO | __GFP_HIGH;
1203 
1204 		rcu_read_lock();
1205 		disk = r10_bio->devs[slot].devnum;
1206 		err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
1207 		if (err_rdev)
1208 			snprintf(b, sizeof(b), "%pg", err_rdev->bdev);
1209 		else {
1210 			strcpy(b, "???");
1211 			/* This never gets dereferenced */
1212 			err_rdev = r10_bio->devs[slot].rdev;
1213 		}
1214 		rcu_read_unlock();
1215 	}
1216 
1217 	if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors))
1218 		return;
1219 	rdev = read_balance(conf, r10_bio, &max_sectors);
1220 	if (!rdev) {
1221 		if (err_rdev) {
1222 			pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
1223 					    mdname(mddev), b,
1224 					    (unsigned long long)r10_bio->sector);
1225 		}
1226 		raid_end_bio_io(r10_bio);
1227 		return;
1228 	}
1229 	if (err_rdev)
1230 		pr_err_ratelimited("md/raid10:%s: %pg: redirecting sector %llu to another mirror\n",
1231 				   mdname(mddev),
1232 				   rdev->bdev,
1233 				   (unsigned long long)r10_bio->sector);
1234 	if (max_sectors < bio_sectors(bio)) {
1235 		struct bio *split = bio_split(bio, max_sectors,
1236 					      gfp, &conf->bio_split);
1237 		bio_chain(split, bio);
1238 		allow_barrier(conf);
1239 		submit_bio_noacct(bio);
1240 		wait_barrier(conf, false);
1241 		bio = split;
1242 		r10_bio->master_bio = bio;
1243 		r10_bio->sectors = max_sectors;
1244 	}
1245 	slot = r10_bio->read_slot;
1246 
1247 	if (blk_queue_io_stat(bio->bi_bdev->bd_disk->queue))
1248 		r10_bio->start_time = bio_start_io_acct(bio);
1249 	read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set);
1250 
1251 	r10_bio->devs[slot].bio = read_bio;
1252 	r10_bio->devs[slot].rdev = rdev;
1253 
1254 	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1255 		choose_data_offset(r10_bio, rdev);
1256 	read_bio->bi_end_io = raid10_end_read_request;
1257 	bio_set_op_attrs(read_bio, op, do_sync);
1258 	if (test_bit(FailFast, &rdev->flags) &&
1259 	    test_bit(R10BIO_FailFast, &r10_bio->state))
1260 	        read_bio->bi_opf |= MD_FAILFAST;
1261 	read_bio->bi_private = r10_bio;
1262 
1263 	if (mddev->gendisk)
1264 	        trace_block_bio_remap(read_bio, disk_devt(mddev->gendisk),
1265 	                              r10_bio->sector);
1266 	submit_bio_noacct(read_bio);
1267 	return;
1268 }
1269 
raid10_write_one_disk(struct mddev * mddev,struct r10bio * r10_bio,struct bio * bio,bool replacement,int n_copy)1270 static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
1271 				  struct bio *bio, bool replacement,
1272 				  int n_copy)
1273 {
1274 	const enum req_op op = bio_op(bio);
1275 	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1276 	const blk_opf_t do_fua = bio->bi_opf & REQ_FUA;
1277 	unsigned long flags;
1278 	struct blk_plug_cb *cb;
1279 	struct raid1_plug_cb *plug = NULL;
1280 	struct r10conf *conf = mddev->private;
1281 	struct md_rdev *rdev;
1282 	int devnum = r10_bio->devs[n_copy].devnum;
1283 	struct bio *mbio;
1284 
1285 	if (replacement) {
1286 		rdev = conf->mirrors[devnum].replacement;
1287 		if (rdev == NULL) {
1288 			/* Replacement just got moved to main 'rdev' */
1289 			smp_mb();
1290 			rdev = conf->mirrors[devnum].rdev;
1291 		}
1292 	} else
1293 		rdev = conf->mirrors[devnum].rdev;
1294 
1295 	mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set);
1296 	if (replacement)
1297 		r10_bio->devs[n_copy].repl_bio = mbio;
1298 	else
1299 		r10_bio->devs[n_copy].bio = mbio;
1300 
1301 	mbio->bi_iter.bi_sector	= (r10_bio->devs[n_copy].addr +
1302 				   choose_data_offset(r10_bio, rdev));
1303 	mbio->bi_end_io	= raid10_end_write_request;
1304 	bio_set_op_attrs(mbio, op, do_sync | do_fua);
1305 	if (!replacement && test_bit(FailFast,
1306 				     &conf->mirrors[devnum].rdev->flags)
1307 			 && enough(conf, devnum))
1308 		mbio->bi_opf |= MD_FAILFAST;
1309 	mbio->bi_private = r10_bio;
1310 
1311 	if (conf->mddev->gendisk)
1312 		trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk),
1313 				      r10_bio->sector);
1314 	/* flush_pending_writes() needs access to the rdev so...*/
1315 	mbio->bi_bdev = (void *)rdev;
1316 
1317 	atomic_inc(&r10_bio->remaining);
1318 
1319 	cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
1320 	if (cb)
1321 		plug = container_of(cb, struct raid1_plug_cb, cb);
1322 	else
1323 		plug = NULL;
1324 	if (plug) {
1325 		bio_list_add(&plug->pending, mbio);
1326 	} else {
1327 		spin_lock_irqsave(&conf->device_lock, flags);
1328 		bio_list_add(&conf->pending_bio_list, mbio);
1329 		spin_unlock_irqrestore(&conf->device_lock, flags);
1330 		md_wakeup_thread(mddev->thread);
1331 	}
1332 }
1333 
wait_blocked_dev(struct mddev * mddev,struct r10bio * r10_bio)1334 static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
1335 {
1336 	int i;
1337 	struct r10conf *conf = mddev->private;
1338 	struct md_rdev *blocked_rdev;
1339 
1340 retry_wait:
1341 	blocked_rdev = NULL;
1342 	rcu_read_lock();
1343 	for (i = 0; i < conf->copies; i++) {
1344 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1345 		struct md_rdev *rrdev = rcu_dereference(
1346 			conf->mirrors[i].replacement);
1347 		if (rdev == rrdev)
1348 			rrdev = NULL;
1349 		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1350 			atomic_inc(&rdev->nr_pending);
1351 			blocked_rdev = rdev;
1352 			break;
1353 		}
1354 		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1355 			atomic_inc(&rrdev->nr_pending);
1356 			blocked_rdev = rrdev;
1357 			break;
1358 		}
1359 
1360 		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1361 			sector_t first_bad;
1362 			sector_t dev_sector = r10_bio->devs[i].addr;
1363 			int bad_sectors;
1364 			int is_bad;
1365 
1366 			/*
1367 			 * Discard request doesn't care the write result
1368 			 * so it doesn't need to wait blocked disk here.
1369 			 */
1370 			if (!r10_bio->sectors)
1371 				continue;
1372 
1373 			is_bad = is_badblock(rdev, dev_sector, r10_bio->sectors,
1374 					     &first_bad, &bad_sectors);
1375 			if (is_bad < 0) {
1376 				/*
1377 				 * Mustn't write here until the bad block
1378 				 * is acknowledged
1379 				 */
1380 				atomic_inc(&rdev->nr_pending);
1381 				set_bit(BlockedBadBlocks, &rdev->flags);
1382 				blocked_rdev = rdev;
1383 				break;
1384 			}
1385 		}
1386 	}
1387 	rcu_read_unlock();
1388 
1389 	if (unlikely(blocked_rdev)) {
1390 		/* Have to wait for this device to get unblocked, then retry */
1391 		allow_barrier(conf);
1392 		raid10_log(conf->mddev, "%s wait rdev %d blocked",
1393 				__func__, blocked_rdev->raid_disk);
1394 		md_wait_for_blocked_rdev(blocked_rdev, mddev);
1395 		wait_barrier(conf, false);
1396 		goto retry_wait;
1397 	}
1398 }
1399 
raid10_write_request(struct mddev * mddev,struct bio * bio,struct r10bio * r10_bio)1400 static void raid10_write_request(struct mddev *mddev, struct bio *bio,
1401 				 struct r10bio *r10_bio)
1402 {
1403 	struct r10conf *conf = mddev->private;
1404 	int i;
1405 	sector_t sectors;
1406 	int max_sectors;
1407 
1408 	if ((mddev_is_clustered(mddev) &&
1409 	     md_cluster_ops->area_resyncing(mddev, WRITE,
1410 					    bio->bi_iter.bi_sector,
1411 					    bio_end_sector(bio)))) {
1412 		DEFINE_WAIT(w);
1413 		/* Bail out if REQ_NOWAIT is set for the bio */
1414 		if (bio->bi_opf & REQ_NOWAIT) {
1415 			bio_wouldblock_error(bio);
1416 			return;
1417 		}
1418 		for (;;) {
1419 			prepare_to_wait(&conf->wait_barrier,
1420 					&w, TASK_IDLE);
1421 			if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1422 				 bio->bi_iter.bi_sector, bio_end_sector(bio)))
1423 				break;
1424 			schedule();
1425 		}
1426 		finish_wait(&conf->wait_barrier, &w);
1427 	}
1428 
1429 	sectors = r10_bio->sectors;
1430 	if (!regular_request_wait(mddev, conf, bio, sectors))
1431 		return;
1432 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1433 	    (mddev->reshape_backwards
1434 	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
1435 		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
1436 	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
1437 		bio->bi_iter.bi_sector < conf->reshape_progress))) {
1438 		/* Need to update reshape_position in metadata */
1439 		mddev->reshape_position = conf->reshape_progress;
1440 		set_mask_bits(&mddev->sb_flags, 0,
1441 			      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1442 		md_wakeup_thread(mddev->thread);
1443 		if (bio->bi_opf & REQ_NOWAIT) {
1444 			allow_barrier(conf);
1445 			bio_wouldblock_error(bio);
1446 			return;
1447 		}
1448 		raid10_log(conf->mddev, "wait reshape metadata");
1449 		wait_event(mddev->sb_wait,
1450 			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1451 
1452 		conf->reshape_safe = mddev->reshape_position;
1453 	}
1454 
1455 	/* first select target devices under rcu_lock and
1456 	 * inc refcount on their rdev.  Record them by setting
1457 	 * bios[x] to bio
1458 	 * If there are known/acknowledged bad blocks on any device
1459 	 * on which we have seen a write error, we want to avoid
1460 	 * writing to those blocks.  This potentially requires several
1461 	 * writes to write around the bad blocks.  Each set of writes
1462 	 * gets its own r10_bio with a set of bios attached.
1463 	 */
1464 
1465 	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1466 	raid10_find_phys(conf, r10_bio);
1467 
1468 	wait_blocked_dev(mddev, r10_bio);
1469 
1470 	rcu_read_lock();
1471 	max_sectors = r10_bio->sectors;
1472 
1473 	for (i = 0;  i < conf->copies; i++) {
1474 		int d = r10_bio->devs[i].devnum;
1475 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1476 		struct md_rdev *rrdev = rcu_dereference(
1477 			conf->mirrors[d].replacement);
1478 		if (rdev == rrdev)
1479 			rrdev = NULL;
1480 		if (rdev && (test_bit(Faulty, &rdev->flags)))
1481 			rdev = NULL;
1482 		if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1483 			rrdev = NULL;
1484 
1485 		r10_bio->devs[i].bio = NULL;
1486 		r10_bio->devs[i].repl_bio = NULL;
1487 
1488 		if (!rdev && !rrdev) {
1489 			set_bit(R10BIO_Degraded, &r10_bio->state);
1490 			continue;
1491 		}
1492 		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1493 			sector_t first_bad;
1494 			sector_t dev_sector = r10_bio->devs[i].addr;
1495 			int bad_sectors;
1496 			int is_bad;
1497 
1498 			is_bad = is_badblock(rdev, dev_sector, max_sectors,
1499 					     &first_bad, &bad_sectors);
1500 			if (is_bad && first_bad <= dev_sector) {
1501 				/* Cannot write here at all */
1502 				bad_sectors -= (dev_sector - first_bad);
1503 				if (bad_sectors < max_sectors)
1504 					/* Mustn't write more than bad_sectors
1505 					 * to other devices yet
1506 					 */
1507 					max_sectors = bad_sectors;
1508 				/* We don't set R10BIO_Degraded as that
1509 				 * only applies if the disk is missing,
1510 				 * so it might be re-added, and we want to
1511 				 * know to recover this chunk.
1512 				 * In this case the device is here, and the
1513 				 * fact that this chunk is not in-sync is
1514 				 * recorded in the bad block log.
1515 				 */
1516 				continue;
1517 			}
1518 			if (is_bad) {
1519 				int good_sectors = first_bad - dev_sector;
1520 				if (good_sectors < max_sectors)
1521 					max_sectors = good_sectors;
1522 			}
1523 		}
1524 		if (rdev) {
1525 			r10_bio->devs[i].bio = bio;
1526 			atomic_inc(&rdev->nr_pending);
1527 		}
1528 		if (rrdev) {
1529 			r10_bio->devs[i].repl_bio = bio;
1530 			atomic_inc(&rrdev->nr_pending);
1531 		}
1532 	}
1533 	rcu_read_unlock();
1534 
1535 	if (max_sectors < r10_bio->sectors)
1536 		r10_bio->sectors = max_sectors;
1537 
1538 	if (r10_bio->sectors < bio_sectors(bio)) {
1539 		struct bio *split = bio_split(bio, r10_bio->sectors,
1540 					      GFP_NOIO, &conf->bio_split);
1541 		bio_chain(split, bio);
1542 		allow_barrier(conf);
1543 		submit_bio_noacct(bio);
1544 		wait_barrier(conf, false);
1545 		bio = split;
1546 		r10_bio->master_bio = bio;
1547 	}
1548 
1549 	if (blk_queue_io_stat(bio->bi_bdev->bd_disk->queue))
1550 		r10_bio->start_time = bio_start_io_acct(bio);
1551 	atomic_set(&r10_bio->remaining, 1);
1552 	md_bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1553 
1554 	for (i = 0; i < conf->copies; i++) {
1555 		if (r10_bio->devs[i].bio)
1556 			raid10_write_one_disk(mddev, r10_bio, bio, false, i);
1557 		if (r10_bio->devs[i].repl_bio)
1558 			raid10_write_one_disk(mddev, r10_bio, bio, true, i);
1559 	}
1560 	one_write_done(r10_bio);
1561 }
1562 
__make_request(struct mddev * mddev,struct bio * bio,int sectors)1563 static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1564 {
1565 	struct r10conf *conf = mddev->private;
1566 	struct r10bio *r10_bio;
1567 
1568 	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
1569 
1570 	r10_bio->master_bio = bio;
1571 	r10_bio->sectors = sectors;
1572 
1573 	r10_bio->mddev = mddev;
1574 	r10_bio->sector = bio->bi_iter.bi_sector;
1575 	r10_bio->state = 0;
1576 	r10_bio->read_slot = -1;
1577 	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *
1578 			conf->geo.raid_disks);
1579 
1580 	if (bio_data_dir(bio) == READ)
1581 		raid10_read_request(mddev, bio, r10_bio);
1582 	else
1583 		raid10_write_request(mddev, bio, r10_bio);
1584 }
1585 
raid_end_discard_bio(struct r10bio * r10bio)1586 static void raid_end_discard_bio(struct r10bio *r10bio)
1587 {
1588 	struct r10conf *conf = r10bio->mddev->private;
1589 	struct r10bio *first_r10bio;
1590 
1591 	while (atomic_dec_and_test(&r10bio->remaining)) {
1592 
1593 		allow_barrier(conf);
1594 
1595 		if (!test_bit(R10BIO_Discard, &r10bio->state)) {
1596 			first_r10bio = (struct r10bio *)r10bio->master_bio;
1597 			free_r10bio(r10bio);
1598 			r10bio = first_r10bio;
1599 		} else {
1600 			md_write_end(r10bio->mddev);
1601 			bio_endio(r10bio->master_bio);
1602 			free_r10bio(r10bio);
1603 			break;
1604 		}
1605 	}
1606 }
1607 
raid10_end_discard_request(struct bio * bio)1608 static void raid10_end_discard_request(struct bio *bio)
1609 {
1610 	struct r10bio *r10_bio = bio->bi_private;
1611 	struct r10conf *conf = r10_bio->mddev->private;
1612 	struct md_rdev *rdev = NULL;
1613 	int dev;
1614 	int slot, repl;
1615 
1616 	/*
1617 	 * We don't care the return value of discard bio
1618 	 */
1619 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
1620 		set_bit(R10BIO_Uptodate, &r10_bio->state);
1621 
1622 	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1623 	if (repl)
1624 		rdev = conf->mirrors[dev].replacement;
1625 	if (!rdev) {
1626 		/*
1627 		 * raid10_remove_disk uses smp_mb to make sure rdev is set to
1628 		 * replacement before setting replacement to NULL. It can read
1629 		 * rdev first without barrier protect even replacment is NULL
1630 		 */
1631 		smp_rmb();
1632 		rdev = conf->mirrors[dev].rdev;
1633 	}
1634 
1635 	raid_end_discard_bio(r10_bio);
1636 	rdev_dec_pending(rdev, conf->mddev);
1637 }
1638 
1639 /*
1640  * There are some limitations to handle discard bio
1641  * 1st, the discard size is bigger than stripe_size*2.
1642  * 2st, if the discard bio spans reshape progress, we use the old way to
1643  * handle discard bio
1644  */
raid10_handle_discard(struct mddev * mddev,struct bio * bio)1645 static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
1646 {
1647 	struct r10conf *conf = mddev->private;
1648 	struct geom *geo = &conf->geo;
1649 	int far_copies = geo->far_copies;
1650 	bool first_copy = true;
1651 	struct r10bio *r10_bio, *first_r10bio;
1652 	struct bio *split;
1653 	int disk;
1654 	sector_t chunk;
1655 	unsigned int stripe_size;
1656 	unsigned int stripe_data_disks;
1657 	sector_t split_size;
1658 	sector_t bio_start, bio_end;
1659 	sector_t first_stripe_index, last_stripe_index;
1660 	sector_t start_disk_offset;
1661 	unsigned int start_disk_index;
1662 	sector_t end_disk_offset;
1663 	unsigned int end_disk_index;
1664 	unsigned int remainder;
1665 
1666 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1667 		return -EAGAIN;
1668 
1669 	if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {
1670 		bio_wouldblock_error(bio);
1671 		return 0;
1672 	}
1673 	wait_barrier(conf, false);
1674 
1675 	/*
1676 	 * Check reshape again to avoid reshape happens after checking
1677 	 * MD_RECOVERY_RESHAPE and before wait_barrier
1678 	 */
1679 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1680 		goto out;
1681 
1682 	if (geo->near_copies)
1683 		stripe_data_disks = geo->raid_disks / geo->near_copies +
1684 					geo->raid_disks % geo->near_copies;
1685 	else
1686 		stripe_data_disks = geo->raid_disks;
1687 
1688 	stripe_size = stripe_data_disks << geo->chunk_shift;
1689 
1690 	bio_start = bio->bi_iter.bi_sector;
1691 	bio_end = bio_end_sector(bio);
1692 
1693 	/*
1694 	 * Maybe one discard bio is smaller than strip size or across one
1695 	 * stripe and discard region is larger than one stripe size. For far
1696 	 * offset layout, if the discard region is not aligned with stripe
1697 	 * size, there is hole when we submit discard bio to member disk.
1698 	 * For simplicity, we only handle discard bio which discard region
1699 	 * is bigger than stripe_size * 2
1700 	 */
1701 	if (bio_sectors(bio) < stripe_size*2)
1702 		goto out;
1703 
1704 	/*
1705 	 * Keep bio aligned with strip size.
1706 	 */
1707 	div_u64_rem(bio_start, stripe_size, &remainder);
1708 	if (remainder) {
1709 		split_size = stripe_size - remainder;
1710 		split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split);
1711 		bio_chain(split, bio);
1712 		allow_barrier(conf);
1713 		/* Resend the fist split part */
1714 		submit_bio_noacct(split);
1715 		wait_barrier(conf, false);
1716 	}
1717 	div_u64_rem(bio_end, stripe_size, &remainder);
1718 	if (remainder) {
1719 		split_size = bio_sectors(bio) - remainder;
1720 		split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split);
1721 		bio_chain(split, bio);
1722 		allow_barrier(conf);
1723 		/* Resend the second split part */
1724 		submit_bio_noacct(bio);
1725 		bio = split;
1726 		wait_barrier(conf, false);
1727 	}
1728 
1729 	bio_start = bio->bi_iter.bi_sector;
1730 	bio_end = bio_end_sector(bio);
1731 
1732 	/*
1733 	 * Raid10 uses chunk as the unit to store data. It's similar like raid0.
1734 	 * One stripe contains the chunks from all member disk (one chunk from
1735 	 * one disk at the same HBA address). For layout detail, see 'man md 4'
1736 	 */
1737 	chunk = bio_start >> geo->chunk_shift;
1738 	chunk *= geo->near_copies;
1739 	first_stripe_index = chunk;
1740 	start_disk_index = sector_div(first_stripe_index, geo->raid_disks);
1741 	if (geo->far_offset)
1742 		first_stripe_index *= geo->far_copies;
1743 	start_disk_offset = (bio_start & geo->chunk_mask) +
1744 				(first_stripe_index << geo->chunk_shift);
1745 
1746 	chunk = bio_end >> geo->chunk_shift;
1747 	chunk *= geo->near_copies;
1748 	last_stripe_index = chunk;
1749 	end_disk_index = sector_div(last_stripe_index, geo->raid_disks);
1750 	if (geo->far_offset)
1751 		last_stripe_index *= geo->far_copies;
1752 	end_disk_offset = (bio_end & geo->chunk_mask) +
1753 				(last_stripe_index << geo->chunk_shift);
1754 
1755 retry_discard:
1756 	r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
1757 	r10_bio->mddev = mddev;
1758 	r10_bio->state = 0;
1759 	r10_bio->sectors = 0;
1760 	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks);
1761 	wait_blocked_dev(mddev, r10_bio);
1762 
1763 	/*
1764 	 * For far layout it needs more than one r10bio to cover all regions.
1765 	 * Inspired by raid10_sync_request, we can use the first r10bio->master_bio
1766 	 * to record the discard bio. Other r10bio->master_bio record the first
1767 	 * r10bio. The first r10bio only release after all other r10bios finish.
1768 	 * The discard bio returns only first r10bio finishes
1769 	 */
1770 	if (first_copy) {
1771 		r10_bio->master_bio = bio;
1772 		set_bit(R10BIO_Discard, &r10_bio->state);
1773 		first_copy = false;
1774 		first_r10bio = r10_bio;
1775 	} else
1776 		r10_bio->master_bio = (struct bio *)first_r10bio;
1777 
1778 	/*
1779 	 * first select target devices under rcu_lock and
1780 	 * inc refcount on their rdev.  Record them by setting
1781 	 * bios[x] to bio
1782 	 */
1783 	rcu_read_lock();
1784 	for (disk = 0; disk < geo->raid_disks; disk++) {
1785 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[disk].rdev);
1786 		struct md_rdev *rrdev = rcu_dereference(
1787 			conf->mirrors[disk].replacement);
1788 
1789 		r10_bio->devs[disk].bio = NULL;
1790 		r10_bio->devs[disk].repl_bio = NULL;
1791 
1792 		if (rdev && (test_bit(Faulty, &rdev->flags)))
1793 			rdev = NULL;
1794 		if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1795 			rrdev = NULL;
1796 		if (!rdev && !rrdev)
1797 			continue;
1798 
1799 		if (rdev) {
1800 			r10_bio->devs[disk].bio = bio;
1801 			atomic_inc(&rdev->nr_pending);
1802 		}
1803 		if (rrdev) {
1804 			r10_bio->devs[disk].repl_bio = bio;
1805 			atomic_inc(&rrdev->nr_pending);
1806 		}
1807 	}
1808 	rcu_read_unlock();
1809 
1810 	atomic_set(&r10_bio->remaining, 1);
1811 	for (disk = 0; disk < geo->raid_disks; disk++) {
1812 		sector_t dev_start, dev_end;
1813 		struct bio *mbio, *rbio = NULL;
1814 
1815 		/*
1816 		 * Now start to calculate the start and end address for each disk.
1817 		 * The space between dev_start and dev_end is the discard region.
1818 		 *
1819 		 * For dev_start, it needs to consider three conditions:
1820 		 * 1st, the disk is before start_disk, you can imagine the disk in
1821 		 * the next stripe. So the dev_start is the start address of next
1822 		 * stripe.
1823 		 * 2st, the disk is after start_disk, it means the disk is at the
1824 		 * same stripe of first disk
1825 		 * 3st, the first disk itself, we can use start_disk_offset directly
1826 		 */
1827 		if (disk < start_disk_index)
1828 			dev_start = (first_stripe_index + 1) * mddev->chunk_sectors;
1829 		else if (disk > start_disk_index)
1830 			dev_start = first_stripe_index * mddev->chunk_sectors;
1831 		else
1832 			dev_start = start_disk_offset;
1833 
1834 		if (disk < end_disk_index)
1835 			dev_end = (last_stripe_index + 1) * mddev->chunk_sectors;
1836 		else if (disk > end_disk_index)
1837 			dev_end = last_stripe_index * mddev->chunk_sectors;
1838 		else
1839 			dev_end = end_disk_offset;
1840 
1841 		/*
1842 		 * It only handles discard bio which size is >= stripe size, so
1843 		 * dev_end > dev_start all the time.
1844 		 * It doesn't need to use rcu lock to get rdev here. We already
1845 		 * add rdev->nr_pending in the first loop.
1846 		 */
1847 		if (r10_bio->devs[disk].bio) {
1848 			struct md_rdev *rdev = conf->mirrors[disk].rdev;
1849 			mbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO,
1850 					       &mddev->bio_set);
1851 			mbio->bi_end_io = raid10_end_discard_request;
1852 			mbio->bi_private = r10_bio;
1853 			r10_bio->devs[disk].bio = mbio;
1854 			r10_bio->devs[disk].devnum = disk;
1855 			atomic_inc(&r10_bio->remaining);
1856 			md_submit_discard_bio(mddev, rdev, mbio,
1857 					dev_start + choose_data_offset(r10_bio, rdev),
1858 					dev_end - dev_start);
1859 			bio_endio(mbio);
1860 		}
1861 		if (r10_bio->devs[disk].repl_bio) {
1862 			struct md_rdev *rrdev = conf->mirrors[disk].replacement;
1863 			rbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO,
1864 					       &mddev->bio_set);
1865 			rbio->bi_end_io = raid10_end_discard_request;
1866 			rbio->bi_private = r10_bio;
1867 			r10_bio->devs[disk].repl_bio = rbio;
1868 			r10_bio->devs[disk].devnum = disk;
1869 			atomic_inc(&r10_bio->remaining);
1870 			md_submit_discard_bio(mddev, rrdev, rbio,
1871 					dev_start + choose_data_offset(r10_bio, rrdev),
1872 					dev_end - dev_start);
1873 			bio_endio(rbio);
1874 		}
1875 	}
1876 
1877 	if (!geo->far_offset && --far_copies) {
1878 		first_stripe_index += geo->stride >> geo->chunk_shift;
1879 		start_disk_offset += geo->stride;
1880 		last_stripe_index += geo->stride >> geo->chunk_shift;
1881 		end_disk_offset += geo->stride;
1882 		atomic_inc(&first_r10bio->remaining);
1883 		raid_end_discard_bio(r10_bio);
1884 		wait_barrier(conf, false);
1885 		goto retry_discard;
1886 	}
1887 
1888 	raid_end_discard_bio(r10_bio);
1889 
1890 	return 0;
1891 out:
1892 	allow_barrier(conf);
1893 	return -EAGAIN;
1894 }
1895 
raid10_make_request(struct mddev * mddev,struct bio * bio)1896 static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
1897 {
1898 	struct r10conf *conf = mddev->private;
1899 	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1900 	int chunk_sects = chunk_mask + 1;
1901 	int sectors = bio_sectors(bio);
1902 
1903 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1904 	    && md_flush_request(mddev, bio))
1905 		return true;
1906 
1907 	if (!md_write_start(mddev, bio))
1908 		return false;
1909 
1910 	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1911 		if (!raid10_handle_discard(mddev, bio))
1912 			return true;
1913 
1914 	/*
1915 	 * If this request crosses a chunk boundary, we need to split
1916 	 * it.
1917 	 */
1918 	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1919 		     sectors > chunk_sects
1920 		     && (conf->geo.near_copies < conf->geo.raid_disks
1921 			 || conf->prev.near_copies <
1922 			 conf->prev.raid_disks)))
1923 		sectors = chunk_sects -
1924 			(bio->bi_iter.bi_sector &
1925 			 (chunk_sects - 1));
1926 	__make_request(mddev, bio, sectors);
1927 
1928 	/* In case raid10d snuck in to freeze_array */
1929 	wake_up_barrier(conf);
1930 	return true;
1931 }
1932 
raid10_status(struct seq_file * seq,struct mddev * mddev)1933 static void raid10_status(struct seq_file *seq, struct mddev *mddev)
1934 {
1935 	struct r10conf *conf = mddev->private;
1936 	int i;
1937 
1938 	if (conf->geo.near_copies < conf->geo.raid_disks)
1939 		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1940 	if (conf->geo.near_copies > 1)
1941 		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1942 	if (conf->geo.far_copies > 1) {
1943 		if (conf->geo.far_offset)
1944 			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1945 		else
1946 			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1947 		if (conf->geo.far_set_size != conf->geo.raid_disks)
1948 			seq_printf(seq, " %d devices per set", conf->geo.far_set_size);
1949 	}
1950 	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1951 					conf->geo.raid_disks - mddev->degraded);
1952 	rcu_read_lock();
1953 	for (i = 0; i < conf->geo.raid_disks; i++) {
1954 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1955 		seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1956 	}
1957 	rcu_read_unlock();
1958 	seq_printf(seq, "]");
1959 }
1960 
1961 /* check if there are enough drives for
1962  * every block to appear on atleast one.
1963  * Don't consider the device numbered 'ignore'
1964  * as we might be about to remove it.
1965  */
_enough(struct r10conf * conf,int previous,int ignore)1966 static int _enough(struct r10conf *conf, int previous, int ignore)
1967 {
1968 	int first = 0;
1969 	int has_enough = 0;
1970 	int disks, ncopies;
1971 	if (previous) {
1972 		disks = conf->prev.raid_disks;
1973 		ncopies = conf->prev.near_copies;
1974 	} else {
1975 		disks = conf->geo.raid_disks;
1976 		ncopies = conf->geo.near_copies;
1977 	}
1978 
1979 	rcu_read_lock();
1980 	do {
1981 		int n = conf->copies;
1982 		int cnt = 0;
1983 		int this = first;
1984 		while (n--) {
1985 			struct md_rdev *rdev;
1986 			if (this != ignore &&
1987 			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
1988 			    test_bit(In_sync, &rdev->flags))
1989 				cnt++;
1990 			this = (this+1) % disks;
1991 		}
1992 		if (cnt == 0)
1993 			goto out;
1994 		first = (first + ncopies) % disks;
1995 	} while (first != 0);
1996 	has_enough = 1;
1997 out:
1998 	rcu_read_unlock();
1999 	return has_enough;
2000 }
2001 
enough(struct r10conf * conf,int ignore)2002 static int enough(struct r10conf *conf, int ignore)
2003 {
2004 	/* when calling 'enough', both 'prev' and 'geo' must
2005 	 * be stable.
2006 	 * This is ensured if ->reconfig_mutex or ->device_lock
2007 	 * is held.
2008 	 */
2009 	return _enough(conf, 0, ignore) &&
2010 		_enough(conf, 1, ignore);
2011 }
2012 
2013 /**
2014  * raid10_error() - RAID10 error handler.
2015  * @mddev: affected md device.
2016  * @rdev: member device to fail.
2017  *
2018  * The routine acknowledges &rdev failure and determines new @mddev state.
2019  * If it failed, then:
2020  *	- &MD_BROKEN flag is set in &mddev->flags.
2021  * Otherwise, it must be degraded:
2022  *	- recovery is interrupted.
2023  *	- &mddev->degraded is bumped.
2024  *
2025  * @rdev is marked as &Faulty excluding case when array is failed and
2026  * &mddev->fail_last_dev is off.
2027  */
raid10_error(struct mddev * mddev,struct md_rdev * rdev)2028 static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
2029 {
2030 	struct r10conf *conf = mddev->private;
2031 	unsigned long flags;
2032 
2033 	spin_lock_irqsave(&conf->device_lock, flags);
2034 
2035 	if (test_bit(In_sync, &rdev->flags) && !enough(conf, rdev->raid_disk)) {
2036 		set_bit(MD_BROKEN, &mddev->flags);
2037 
2038 		if (!mddev->fail_last_dev) {
2039 			spin_unlock_irqrestore(&conf->device_lock, flags);
2040 			return;
2041 		}
2042 	}
2043 	if (test_and_clear_bit(In_sync, &rdev->flags))
2044 		mddev->degraded++;
2045 
2046 	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2047 	set_bit(Blocked, &rdev->flags);
2048 	set_bit(Faulty, &rdev->flags);
2049 	set_mask_bits(&mddev->sb_flags, 0,
2050 		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
2051 	spin_unlock_irqrestore(&conf->device_lock, flags);
2052 	pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n"
2053 		"md/raid10:%s: Operation continuing on %d devices.\n",
2054 		mdname(mddev), rdev->bdev,
2055 		mdname(mddev), conf->geo.raid_disks - mddev->degraded);
2056 }
2057 
print_conf(struct r10conf * conf)2058 static void print_conf(struct r10conf *conf)
2059 {
2060 	int i;
2061 	struct md_rdev *rdev;
2062 
2063 	pr_debug("RAID10 conf printout:\n");
2064 	if (!conf) {
2065 		pr_debug("(!conf)\n");
2066 		return;
2067 	}
2068 	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
2069 		 conf->geo.raid_disks);
2070 
2071 	/* This is only called with ->reconfix_mutex held, so
2072 	 * rcu protection of rdev is not needed */
2073 	for (i = 0; i < conf->geo.raid_disks; i++) {
2074 		rdev = conf->mirrors[i].rdev;
2075 		if (rdev)
2076 			pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",
2077 				 i, !test_bit(In_sync, &rdev->flags),
2078 				 !test_bit(Faulty, &rdev->flags),
2079 				 rdev->bdev);
2080 	}
2081 }
2082 
close_sync(struct r10conf * conf)2083 static void close_sync(struct r10conf *conf)
2084 {
2085 	wait_barrier(conf, false);
2086 	allow_barrier(conf);
2087 
2088 	mempool_exit(&conf->r10buf_pool);
2089 }
2090 
raid10_spare_active(struct mddev * mddev)2091 static int raid10_spare_active(struct mddev *mddev)
2092 {
2093 	int i;
2094 	struct r10conf *conf = mddev->private;
2095 	struct raid10_info *tmp;
2096 	int count = 0;
2097 	unsigned long flags;
2098 
2099 	/*
2100 	 * Find all non-in_sync disks within the RAID10 configuration
2101 	 * and mark them in_sync
2102 	 */
2103 	for (i = 0; i < conf->geo.raid_disks; i++) {
2104 		tmp = conf->mirrors + i;
2105 		if (tmp->replacement
2106 		    && tmp->replacement->recovery_offset == MaxSector
2107 		    && !test_bit(Faulty, &tmp->replacement->flags)
2108 		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
2109 			/* Replacement has just become active */
2110 			if (!tmp->rdev
2111 			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
2112 				count++;
2113 			if (tmp->rdev) {
2114 				/* Replaced device not technically faulty,
2115 				 * but we need to be sure it gets removed
2116 				 * and never re-added.
2117 				 */
2118 				set_bit(Faulty, &tmp->rdev->flags);
2119 				sysfs_notify_dirent_safe(
2120 					tmp->rdev->sysfs_state);
2121 			}
2122 			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
2123 		} else if (tmp->rdev
2124 			   && tmp->rdev->recovery_offset == MaxSector
2125 			   && !test_bit(Faulty, &tmp->rdev->flags)
2126 			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
2127 			count++;
2128 			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
2129 		}
2130 	}
2131 	spin_lock_irqsave(&conf->device_lock, flags);
2132 	mddev->degraded -= count;
2133 	spin_unlock_irqrestore(&conf->device_lock, flags);
2134 
2135 	print_conf(conf);
2136 	return count;
2137 }
2138 
raid10_add_disk(struct mddev * mddev,struct md_rdev * rdev)2139 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
2140 {
2141 	struct r10conf *conf = mddev->private;
2142 	int err = -EEXIST;
2143 	int mirror;
2144 	int first = 0;
2145 	int last = conf->geo.raid_disks - 1;
2146 
2147 	if (mddev->recovery_cp < MaxSector)
2148 		/* only hot-add to in-sync arrays, as recovery is
2149 		 * very different from resync
2150 		 */
2151 		return -EBUSY;
2152 	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
2153 		return -EINVAL;
2154 
2155 	if (md_integrity_add_rdev(rdev, mddev))
2156 		return -ENXIO;
2157 
2158 	if (rdev->raid_disk >= 0)
2159 		first = last = rdev->raid_disk;
2160 
2161 	if (rdev->saved_raid_disk >= first &&
2162 	    rdev->saved_raid_disk < conf->geo.raid_disks &&
2163 	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
2164 		mirror = rdev->saved_raid_disk;
2165 	else
2166 		mirror = first;
2167 	for ( ; mirror <= last ; mirror++) {
2168 		struct raid10_info *p = &conf->mirrors[mirror];
2169 		if (p->recovery_disabled == mddev->recovery_disabled)
2170 			continue;
2171 		if (p->rdev) {
2172 			if (!test_bit(WantReplacement, &p->rdev->flags) ||
2173 			    p->replacement != NULL)
2174 				continue;
2175 			clear_bit(In_sync, &rdev->flags);
2176 			set_bit(Replacement, &rdev->flags);
2177 			rdev->raid_disk = mirror;
2178 			err = 0;
2179 			if (mddev->gendisk)
2180 				disk_stack_limits(mddev->gendisk, rdev->bdev,
2181 						  rdev->data_offset << 9);
2182 			conf->fullsync = 1;
2183 			rcu_assign_pointer(p->replacement, rdev);
2184 			break;
2185 		}
2186 
2187 		if (mddev->gendisk)
2188 			disk_stack_limits(mddev->gendisk, rdev->bdev,
2189 					  rdev->data_offset << 9);
2190 
2191 		p->head_position = 0;
2192 		p->recovery_disabled = mddev->recovery_disabled - 1;
2193 		rdev->raid_disk = mirror;
2194 		err = 0;
2195 		if (rdev->saved_raid_disk != mirror)
2196 			conf->fullsync = 1;
2197 		rcu_assign_pointer(p->rdev, rdev);
2198 		break;
2199 	}
2200 
2201 	print_conf(conf);
2202 	return err;
2203 }
2204 
raid10_remove_disk(struct mddev * mddev,struct md_rdev * rdev)2205 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
2206 {
2207 	struct r10conf *conf = mddev->private;
2208 	int err = 0;
2209 	int number = rdev->raid_disk;
2210 	struct md_rdev **rdevp;
2211 	struct raid10_info *p;
2212 
2213 	print_conf(conf);
2214 	if (unlikely(number >= mddev->raid_disks))
2215 		return 0;
2216 	p = conf->mirrors + number;
2217 	if (rdev == p->rdev)
2218 		rdevp = &p->rdev;
2219 	else if (rdev == p->replacement)
2220 		rdevp = &p->replacement;
2221 	else
2222 		return 0;
2223 
2224 	if (test_bit(In_sync, &rdev->flags) ||
2225 	    atomic_read(&rdev->nr_pending)) {
2226 		err = -EBUSY;
2227 		goto abort;
2228 	}
2229 	/* Only remove non-faulty devices if recovery
2230 	 * is not possible.
2231 	 */
2232 	if (!test_bit(Faulty, &rdev->flags) &&
2233 	    mddev->recovery_disabled != p->recovery_disabled &&
2234 	    (!p->replacement || p->replacement == rdev) &&
2235 	    number < conf->geo.raid_disks &&
2236 	    enough(conf, -1)) {
2237 		err = -EBUSY;
2238 		goto abort;
2239 	}
2240 	*rdevp = NULL;
2241 	if (!test_bit(RemoveSynchronized, &rdev->flags)) {
2242 		synchronize_rcu();
2243 		if (atomic_read(&rdev->nr_pending)) {
2244 			/* lost the race, try later */
2245 			err = -EBUSY;
2246 			*rdevp = rdev;
2247 			goto abort;
2248 		}
2249 	}
2250 	if (p->replacement) {
2251 		/* We must have just cleared 'rdev' */
2252 		p->rdev = p->replacement;
2253 		clear_bit(Replacement, &p->replacement->flags);
2254 		smp_mb(); /* Make sure other CPUs may see both as identical
2255 			   * but will never see neither -- if they are careful.
2256 			   */
2257 		p->replacement = NULL;
2258 	}
2259 
2260 	clear_bit(WantReplacement, &rdev->flags);
2261 	err = md_integrity_register(mddev);
2262 
2263 abort:
2264 
2265 	print_conf(conf);
2266 	return err;
2267 }
2268 
__end_sync_read(struct r10bio * r10_bio,struct bio * bio,int d)2269 static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
2270 {
2271 	struct r10conf *conf = r10_bio->mddev->private;
2272 
2273 	if (!bio->bi_status)
2274 		set_bit(R10BIO_Uptodate, &r10_bio->state);
2275 	else
2276 		/* The write handler will notice the lack of
2277 		 * R10BIO_Uptodate and record any errors etc
2278 		 */
2279 		atomic_add(r10_bio->sectors,
2280 			   &conf->mirrors[d].rdev->corrected_errors);
2281 
2282 	/* for reconstruct, we always reschedule after a read.
2283 	 * for resync, only after all reads
2284 	 */
2285 	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
2286 	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
2287 	    atomic_dec_and_test(&r10_bio->remaining)) {
2288 		/* we have read all the blocks,
2289 		 * do the comparison in process context in raid10d
2290 		 */
2291 		reschedule_retry(r10_bio);
2292 	}
2293 }
2294 
end_sync_read(struct bio * bio)2295 static void end_sync_read(struct bio *bio)
2296 {
2297 	struct r10bio *r10_bio = get_resync_r10bio(bio);
2298 	struct r10conf *conf = r10_bio->mddev->private;
2299 	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
2300 
2301 	__end_sync_read(r10_bio, bio, d);
2302 }
2303 
end_reshape_read(struct bio * bio)2304 static void end_reshape_read(struct bio *bio)
2305 {
2306 	/* reshape read bio isn't allocated from r10buf_pool */
2307 	struct r10bio *r10_bio = bio->bi_private;
2308 
2309 	__end_sync_read(r10_bio, bio, r10_bio->read_slot);
2310 }
2311 
end_sync_request(struct r10bio * r10_bio)2312 static void end_sync_request(struct r10bio *r10_bio)
2313 {
2314 	struct mddev *mddev = r10_bio->mddev;
2315 
2316 	while (atomic_dec_and_test(&r10_bio->remaining)) {
2317 		if (r10_bio->master_bio == NULL) {
2318 			/* the primary of several recovery bios */
2319 			sector_t s = r10_bio->sectors;
2320 			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2321 			    test_bit(R10BIO_WriteError, &r10_bio->state))
2322 				reschedule_retry(r10_bio);
2323 			else
2324 				put_buf(r10_bio);
2325 			md_done_sync(mddev, s, 1);
2326 			break;
2327 		} else {
2328 			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
2329 			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2330 			    test_bit(R10BIO_WriteError, &r10_bio->state))
2331 				reschedule_retry(r10_bio);
2332 			else
2333 				put_buf(r10_bio);
2334 			r10_bio = r10_bio2;
2335 		}
2336 	}
2337 }
2338 
end_sync_write(struct bio * bio)2339 static void end_sync_write(struct bio *bio)
2340 {
2341 	struct r10bio *r10_bio = get_resync_r10bio(bio);
2342 	struct mddev *mddev = r10_bio->mddev;
2343 	struct r10conf *conf = mddev->private;
2344 	int d;
2345 	sector_t first_bad;
2346 	int bad_sectors;
2347 	int slot;
2348 	int repl;
2349 	struct md_rdev *rdev = NULL;
2350 
2351 	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
2352 	if (repl)
2353 		rdev = conf->mirrors[d].replacement;
2354 	else
2355 		rdev = conf->mirrors[d].rdev;
2356 
2357 	if (bio->bi_status) {
2358 		if (repl)
2359 			md_error(mddev, rdev);
2360 		else {
2361 			set_bit(WriteErrorSeen, &rdev->flags);
2362 			if (!test_and_set_bit(WantReplacement, &rdev->flags))
2363 				set_bit(MD_RECOVERY_NEEDED,
2364 					&rdev->mddev->recovery);
2365 			set_bit(R10BIO_WriteError, &r10_bio->state);
2366 		}
2367 	} else if (is_badblock(rdev,
2368 			     r10_bio->devs[slot].addr,
2369 			     r10_bio->sectors,
2370 			     &first_bad, &bad_sectors))
2371 		set_bit(R10BIO_MadeGood, &r10_bio->state);
2372 
2373 	rdev_dec_pending(rdev, mddev);
2374 
2375 	end_sync_request(r10_bio);
2376 }
2377 
2378 /*
2379  * Note: sync and recover and handled very differently for raid10
2380  * This code is for resync.
2381  * For resync, we read through virtual addresses and read all blocks.
2382  * If there is any error, we schedule a write.  The lowest numbered
2383  * drive is authoritative.
2384  * However requests come for physical address, so we need to map.
2385  * For every physical address there are raid_disks/copies virtual addresses,
2386  * which is always are least one, but is not necessarly an integer.
2387  * This means that a physical address can span multiple chunks, so we may
2388  * have to submit multiple io requests for a single sync request.
2389  */
2390 /*
2391  * We check if all blocks are in-sync and only write to blocks that
2392  * aren't in sync
2393  */
sync_request_write(struct mddev * mddev,struct r10bio * r10_bio)2394 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2395 {
2396 	struct r10conf *conf = mddev->private;
2397 	int i, first;
2398 	struct bio *tbio, *fbio;
2399 	int vcnt;
2400 	struct page **tpages, **fpages;
2401 
2402 	atomic_set(&r10_bio->remaining, 1);
2403 
2404 	/* find the first device with a block */
2405 	for (i=0; i<conf->copies; i++)
2406 		if (!r10_bio->devs[i].bio->bi_status)
2407 			break;
2408 
2409 	if (i == conf->copies)
2410 		goto done;
2411 
2412 	first = i;
2413 	fbio = r10_bio->devs[i].bio;
2414 	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2415 	fbio->bi_iter.bi_idx = 0;
2416 	fpages = get_resync_pages(fbio)->pages;
2417 
2418 	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2419 	/* now find blocks with errors */
2420 	for (i=0 ; i < conf->copies ; i++) {
2421 		int  j, d;
2422 		struct md_rdev *rdev;
2423 		struct resync_pages *rp;
2424 
2425 		tbio = r10_bio->devs[i].bio;
2426 
2427 		if (tbio->bi_end_io != end_sync_read)
2428 			continue;
2429 		if (i == first)
2430 			continue;
2431 
2432 		tpages = get_resync_pages(tbio)->pages;
2433 		d = r10_bio->devs[i].devnum;
2434 		rdev = conf->mirrors[d].rdev;
2435 		if (!r10_bio->devs[i].bio->bi_status) {
2436 			/* We know that the bi_io_vec layout is the same for
2437 			 * both 'first' and 'i', so we just compare them.
2438 			 * All vec entries are PAGE_SIZE;
2439 			 */
2440 			int sectors = r10_bio->sectors;
2441 			for (j = 0; j < vcnt; j++) {
2442 				int len = PAGE_SIZE;
2443 				if (sectors < (len / 512))
2444 					len = sectors * 512;
2445 				if (memcmp(page_address(fpages[j]),
2446 					   page_address(tpages[j]),
2447 					   len))
2448 					break;
2449 				sectors -= len/512;
2450 			}
2451 			if (j == vcnt)
2452 				continue;
2453 			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2454 			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2455 				/* Don't fix anything. */
2456 				continue;
2457 		} else if (test_bit(FailFast, &rdev->flags)) {
2458 			/* Just give up on this device */
2459 			md_error(rdev->mddev, rdev);
2460 			continue;
2461 		}
2462 		/* Ok, we need to write this bio, either to correct an
2463 		 * inconsistency or to correct an unreadable block.
2464 		 * First we need to fixup bv_offset, bv_len and
2465 		 * bi_vecs, as the read request might have corrupted these
2466 		 */
2467 		rp = get_resync_pages(tbio);
2468 		bio_reset(tbio, conf->mirrors[d].rdev->bdev, REQ_OP_WRITE);
2469 
2470 		md_bio_reset_resync_pages(tbio, rp, fbio->bi_iter.bi_size);
2471 
2472 		rp->raid_bio = r10_bio;
2473 		tbio->bi_private = rp;
2474 		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2475 		tbio->bi_end_io = end_sync_write;
2476 
2477 		bio_copy_data(tbio, fbio);
2478 
2479 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2480 		atomic_inc(&r10_bio->remaining);
2481 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
2482 
2483 		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
2484 			tbio->bi_opf |= MD_FAILFAST;
2485 		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2486 		submit_bio_noacct(tbio);
2487 	}
2488 
2489 	/* Now write out to any replacement devices
2490 	 * that are active
2491 	 */
2492 	for (i = 0; i < conf->copies; i++) {
2493 		int d;
2494 
2495 		tbio = r10_bio->devs[i].repl_bio;
2496 		if (!tbio || !tbio->bi_end_io)
2497 			continue;
2498 		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2499 		    && r10_bio->devs[i].bio != fbio)
2500 			bio_copy_data(tbio, fbio);
2501 		d = r10_bio->devs[i].devnum;
2502 		atomic_inc(&r10_bio->remaining);
2503 		md_sync_acct(conf->mirrors[d].replacement->bdev,
2504 			     bio_sectors(tbio));
2505 		submit_bio_noacct(tbio);
2506 	}
2507 
2508 done:
2509 	if (atomic_dec_and_test(&r10_bio->remaining)) {
2510 		md_done_sync(mddev, r10_bio->sectors, 1);
2511 		put_buf(r10_bio);
2512 	}
2513 }
2514 
2515 /*
2516  * Now for the recovery code.
2517  * Recovery happens across physical sectors.
2518  * We recover all non-is_sync drives by finding the virtual address of
2519  * each, and then choose a working drive that also has that virt address.
2520  * There is a separate r10_bio for each non-in_sync drive.
2521  * Only the first two slots are in use. The first for reading,
2522  * The second for writing.
2523  *
2524  */
fix_recovery_read_error(struct r10bio * r10_bio)2525 static void fix_recovery_read_error(struct r10bio *r10_bio)
2526 {
2527 	/* We got a read error during recovery.
2528 	 * We repeat the read in smaller page-sized sections.
2529 	 * If a read succeeds, write it to the new device or record
2530 	 * a bad block if we cannot.
2531 	 * If a read fails, record a bad block on both old and
2532 	 * new devices.
2533 	 */
2534 	struct mddev *mddev = r10_bio->mddev;
2535 	struct r10conf *conf = mddev->private;
2536 	struct bio *bio = r10_bio->devs[0].bio;
2537 	sector_t sect = 0;
2538 	int sectors = r10_bio->sectors;
2539 	int idx = 0;
2540 	int dr = r10_bio->devs[0].devnum;
2541 	int dw = r10_bio->devs[1].devnum;
2542 	struct page **pages = get_resync_pages(bio)->pages;
2543 
2544 	while (sectors) {
2545 		int s = sectors;
2546 		struct md_rdev *rdev;
2547 		sector_t addr;
2548 		int ok;
2549 
2550 		if (s > (PAGE_SIZE>>9))
2551 			s = PAGE_SIZE >> 9;
2552 
2553 		rdev = conf->mirrors[dr].rdev;
2554 		addr = r10_bio->devs[0].addr + sect,
2555 		ok = sync_page_io(rdev,
2556 				  addr,
2557 				  s << 9,
2558 				  pages[idx],
2559 				  REQ_OP_READ, false);
2560 		if (ok) {
2561 			rdev = conf->mirrors[dw].rdev;
2562 			addr = r10_bio->devs[1].addr + sect;
2563 			ok = sync_page_io(rdev,
2564 					  addr,
2565 					  s << 9,
2566 					  pages[idx],
2567 					  REQ_OP_WRITE, false);
2568 			if (!ok) {
2569 				set_bit(WriteErrorSeen, &rdev->flags);
2570 				if (!test_and_set_bit(WantReplacement,
2571 						      &rdev->flags))
2572 					set_bit(MD_RECOVERY_NEEDED,
2573 						&rdev->mddev->recovery);
2574 			}
2575 		}
2576 		if (!ok) {
2577 			/* We don't worry if we cannot set a bad block -
2578 			 * it really is bad so there is no loss in not
2579 			 * recording it yet
2580 			 */
2581 			rdev_set_badblocks(rdev, addr, s, 0);
2582 
2583 			if (rdev != conf->mirrors[dw].rdev) {
2584 				/* need bad block on destination too */
2585 				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2586 				addr = r10_bio->devs[1].addr + sect;
2587 				ok = rdev_set_badblocks(rdev2, addr, s, 0);
2588 				if (!ok) {
2589 					/* just abort the recovery */
2590 					pr_notice("md/raid10:%s: recovery aborted due to read error\n",
2591 						  mdname(mddev));
2592 
2593 					conf->mirrors[dw].recovery_disabled
2594 						= mddev->recovery_disabled;
2595 					set_bit(MD_RECOVERY_INTR,
2596 						&mddev->recovery);
2597 					break;
2598 				}
2599 			}
2600 		}
2601 
2602 		sectors -= s;
2603 		sect += s;
2604 		idx++;
2605 	}
2606 }
2607 
recovery_request_write(struct mddev * mddev,struct r10bio * r10_bio)2608 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2609 {
2610 	struct r10conf *conf = mddev->private;
2611 	int d;
2612 	struct bio *wbio, *wbio2;
2613 
2614 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2615 		fix_recovery_read_error(r10_bio);
2616 		end_sync_request(r10_bio);
2617 		return;
2618 	}
2619 
2620 	/*
2621 	 * share the pages with the first bio
2622 	 * and submit the write request
2623 	 */
2624 	d = r10_bio->devs[1].devnum;
2625 	wbio = r10_bio->devs[1].bio;
2626 	wbio2 = r10_bio->devs[1].repl_bio;
2627 	/* Need to test wbio2->bi_end_io before we call
2628 	 * submit_bio_noacct as if the former is NULL,
2629 	 * the latter is free to free wbio2.
2630 	 */
2631 	if (wbio2 && !wbio2->bi_end_io)
2632 		wbio2 = NULL;
2633 	if (wbio->bi_end_io) {
2634 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2635 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2636 		submit_bio_noacct(wbio);
2637 	}
2638 	if (wbio2) {
2639 		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2640 		md_sync_acct(conf->mirrors[d].replacement->bdev,
2641 			     bio_sectors(wbio2));
2642 		submit_bio_noacct(wbio2);
2643 	}
2644 }
2645 
2646 /*
2647  * Used by fix_read_error() to decay the per rdev read_errors.
2648  * We halve the read error count for every hour that has elapsed
2649  * since the last recorded read error.
2650  *
2651  */
check_decay_read_errors(struct mddev * mddev,struct md_rdev * rdev)2652 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2653 {
2654 	long cur_time_mon;
2655 	unsigned long hours_since_last;
2656 	unsigned int read_errors = atomic_read(&rdev->read_errors);
2657 
2658 	cur_time_mon = ktime_get_seconds();
2659 
2660 	if (rdev->last_read_error == 0) {
2661 		/* first time we've seen a read error */
2662 		rdev->last_read_error = cur_time_mon;
2663 		return;
2664 	}
2665 
2666 	hours_since_last = (long)(cur_time_mon -
2667 			    rdev->last_read_error) / 3600;
2668 
2669 	rdev->last_read_error = cur_time_mon;
2670 
2671 	/*
2672 	 * if hours_since_last is > the number of bits in read_errors
2673 	 * just set read errors to 0. We do this to avoid
2674 	 * overflowing the shift of read_errors by hours_since_last.
2675 	 */
2676 	if (hours_since_last >= 8 * sizeof(read_errors))
2677 		atomic_set(&rdev->read_errors, 0);
2678 	else
2679 		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2680 }
2681 
r10_sync_page_io(struct md_rdev * rdev,sector_t sector,int sectors,struct page * page,enum req_op op)2682 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2683 			    int sectors, struct page *page, enum req_op op)
2684 {
2685 	sector_t first_bad;
2686 	int bad_sectors;
2687 
2688 	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2689 	    && (op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags)))
2690 		return -1;
2691 	if (sync_page_io(rdev, sector, sectors << 9, page, op, false))
2692 		/* success */
2693 		return 1;
2694 	if (op == REQ_OP_WRITE) {
2695 		set_bit(WriteErrorSeen, &rdev->flags);
2696 		if (!test_and_set_bit(WantReplacement, &rdev->flags))
2697 			set_bit(MD_RECOVERY_NEEDED,
2698 				&rdev->mddev->recovery);
2699 	}
2700 	/* need to record an error - either for the block or the device */
2701 	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2702 		md_error(rdev->mddev, rdev);
2703 	return 0;
2704 }
2705 
2706 /*
2707  * This is a kernel thread which:
2708  *
2709  *	1.	Retries failed read operations on working mirrors.
2710  *	2.	Updates the raid superblock when problems encounter.
2711  *	3.	Performs writes following reads for array synchronising.
2712  */
2713 
fix_read_error(struct r10conf * conf,struct mddev * mddev,struct r10bio * r10_bio)2714 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2715 {
2716 	int sect = 0; /* Offset from r10_bio->sector */
2717 	int sectors = r10_bio->sectors;
2718 	struct md_rdev *rdev;
2719 	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2720 	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2721 
2722 	/* still own a reference to this rdev, so it cannot
2723 	 * have been cleared recently.
2724 	 */
2725 	rdev = conf->mirrors[d].rdev;
2726 
2727 	if (test_bit(Faulty, &rdev->flags))
2728 		/* drive has already been failed, just ignore any
2729 		   more fix_read_error() attempts */
2730 		return;
2731 
2732 	check_decay_read_errors(mddev, rdev);
2733 	atomic_inc(&rdev->read_errors);
2734 	if (atomic_read(&rdev->read_errors) > max_read_errors) {
2735 		pr_notice("md/raid10:%s: %pg: Raid device exceeded read_error threshold [cur %d:max %d]\n",
2736 			  mdname(mddev), rdev->bdev,
2737 			  atomic_read(&rdev->read_errors), max_read_errors);
2738 		pr_notice("md/raid10:%s: %pg: Failing raid device\n",
2739 			  mdname(mddev), rdev->bdev);
2740 		md_error(mddev, rdev);
2741 		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2742 		return;
2743 	}
2744 
2745 	while(sectors) {
2746 		int s = sectors;
2747 		int sl = r10_bio->read_slot;
2748 		int success = 0;
2749 		int start;
2750 
2751 		if (s > (PAGE_SIZE>>9))
2752 			s = PAGE_SIZE >> 9;
2753 
2754 		rcu_read_lock();
2755 		do {
2756 			sector_t first_bad;
2757 			int bad_sectors;
2758 
2759 			d = r10_bio->devs[sl].devnum;
2760 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2761 			if (rdev &&
2762 			    test_bit(In_sync, &rdev->flags) &&
2763 			    !test_bit(Faulty, &rdev->flags) &&
2764 			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2765 					&first_bad, &bad_sectors) == 0) {
2766 				atomic_inc(&rdev->nr_pending);
2767 				rcu_read_unlock();
2768 				success = sync_page_io(rdev,
2769 						       r10_bio->devs[sl].addr +
2770 						       sect,
2771 						       s<<9,
2772 						       conf->tmppage,
2773 						       REQ_OP_READ, false);
2774 				rdev_dec_pending(rdev, mddev);
2775 				rcu_read_lock();
2776 				if (success)
2777 					break;
2778 			}
2779 			sl++;
2780 			if (sl == conf->copies)
2781 				sl = 0;
2782 		} while (!success && sl != r10_bio->read_slot);
2783 		rcu_read_unlock();
2784 
2785 		if (!success) {
2786 			/* Cannot read from anywhere, just mark the block
2787 			 * as bad on the first device to discourage future
2788 			 * reads.
2789 			 */
2790 			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2791 			rdev = conf->mirrors[dn].rdev;
2792 
2793 			if (!rdev_set_badblocks(
2794 				    rdev,
2795 				    r10_bio->devs[r10_bio->read_slot].addr
2796 				    + sect,
2797 				    s, 0)) {
2798 				md_error(mddev, rdev);
2799 				r10_bio->devs[r10_bio->read_slot].bio
2800 					= IO_BLOCKED;
2801 			}
2802 			break;
2803 		}
2804 
2805 		start = sl;
2806 		/* write it back and re-read */
2807 		rcu_read_lock();
2808 		while (sl != r10_bio->read_slot) {
2809 			if (sl==0)
2810 				sl = conf->copies;
2811 			sl--;
2812 			d = r10_bio->devs[sl].devnum;
2813 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2814 			if (!rdev ||
2815 			    test_bit(Faulty, &rdev->flags) ||
2816 			    !test_bit(In_sync, &rdev->flags))
2817 				continue;
2818 
2819 			atomic_inc(&rdev->nr_pending);
2820 			rcu_read_unlock();
2821 			if (r10_sync_page_io(rdev,
2822 					     r10_bio->devs[sl].addr +
2823 					     sect,
2824 					     s, conf->tmppage, REQ_OP_WRITE)
2825 			    == 0) {
2826 				/* Well, this device is dead */
2827 				pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n",
2828 					  mdname(mddev), s,
2829 					  (unsigned long long)(
2830 						  sect +
2831 						  choose_data_offset(r10_bio,
2832 								     rdev)),
2833 					  rdev->bdev);
2834 				pr_notice("md/raid10:%s: %pg: failing drive\n",
2835 					  mdname(mddev),
2836 					  rdev->bdev);
2837 			}
2838 			rdev_dec_pending(rdev, mddev);
2839 			rcu_read_lock();
2840 		}
2841 		sl = start;
2842 		while (sl != r10_bio->read_slot) {
2843 			if (sl==0)
2844 				sl = conf->copies;
2845 			sl--;
2846 			d = r10_bio->devs[sl].devnum;
2847 			rdev = rcu_dereference(conf->mirrors[d].rdev);
2848 			if (!rdev ||
2849 			    test_bit(Faulty, &rdev->flags) ||
2850 			    !test_bit(In_sync, &rdev->flags))
2851 				continue;
2852 
2853 			atomic_inc(&rdev->nr_pending);
2854 			rcu_read_unlock();
2855 			switch (r10_sync_page_io(rdev,
2856 					     r10_bio->devs[sl].addr +
2857 					     sect,
2858 					     s, conf->tmppage, REQ_OP_READ)) {
2859 			case 0:
2860 				/* Well, this device is dead */
2861 				pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n",
2862 				       mdname(mddev), s,
2863 				       (unsigned long long)(
2864 					       sect +
2865 					       choose_data_offset(r10_bio, rdev)),
2866 				       rdev->bdev);
2867 				pr_notice("md/raid10:%s: %pg: failing drive\n",
2868 				       mdname(mddev),
2869 				       rdev->bdev);
2870 				break;
2871 			case 1:
2872 				pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %pg)\n",
2873 				       mdname(mddev), s,
2874 				       (unsigned long long)(
2875 					       sect +
2876 					       choose_data_offset(r10_bio, rdev)),
2877 				       rdev->bdev);
2878 				atomic_add(s, &rdev->corrected_errors);
2879 			}
2880 
2881 			rdev_dec_pending(rdev, mddev);
2882 			rcu_read_lock();
2883 		}
2884 		rcu_read_unlock();
2885 
2886 		sectors -= s;
2887 		sect += s;
2888 	}
2889 }
2890 
narrow_write_error(struct r10bio * r10_bio,int i)2891 static int narrow_write_error(struct r10bio *r10_bio, int i)
2892 {
2893 	struct bio *bio = r10_bio->master_bio;
2894 	struct mddev *mddev = r10_bio->mddev;
2895 	struct r10conf *conf = mddev->private;
2896 	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2897 	/* bio has the data to be written to slot 'i' where
2898 	 * we just recently had a write error.
2899 	 * We repeatedly clone the bio and trim down to one block,
2900 	 * then try the write.  Where the write fails we record
2901 	 * a bad block.
2902 	 * It is conceivable that the bio doesn't exactly align with
2903 	 * blocks.  We must handle this.
2904 	 *
2905 	 * We currently own a reference to the rdev.
2906 	 */
2907 
2908 	int block_sectors;
2909 	sector_t sector;
2910 	int sectors;
2911 	int sect_to_write = r10_bio->sectors;
2912 	int ok = 1;
2913 
2914 	if (rdev->badblocks.shift < 0)
2915 		return 0;
2916 
2917 	block_sectors = roundup(1 << rdev->badblocks.shift,
2918 				bdev_logical_block_size(rdev->bdev) >> 9);
2919 	sector = r10_bio->sector;
2920 	sectors = ((r10_bio->sector + block_sectors)
2921 		   & ~(sector_t)(block_sectors - 1))
2922 		- sector;
2923 
2924 	while (sect_to_write) {
2925 		struct bio *wbio;
2926 		sector_t wsector;
2927 		if (sectors > sect_to_write)
2928 			sectors = sect_to_write;
2929 		/* Write at 'sector' for 'sectors' */
2930 		wbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO,
2931 				       &mddev->bio_set);
2932 		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
2933 		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
2934 		wbio->bi_iter.bi_sector = wsector +
2935 				   choose_data_offset(r10_bio, rdev);
2936 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2937 
2938 		if (submit_bio_wait(wbio) < 0)
2939 			/* Failure! */
2940 			ok = rdev_set_badblocks(rdev, wsector,
2941 						sectors, 0)
2942 				&& ok;
2943 
2944 		bio_put(wbio);
2945 		sect_to_write -= sectors;
2946 		sector += sectors;
2947 		sectors = block_sectors;
2948 	}
2949 	return ok;
2950 }
2951 
handle_read_error(struct mddev * mddev,struct r10bio * r10_bio)2952 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2953 {
2954 	int slot = r10_bio->read_slot;
2955 	struct bio *bio;
2956 	struct r10conf *conf = mddev->private;
2957 	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2958 
2959 	/* we got a read error. Maybe the drive is bad.  Maybe just
2960 	 * the block and we can fix it.
2961 	 * We freeze all other IO, and try reading the block from
2962 	 * other devices.  When we find one, we re-write
2963 	 * and check it that fixes the read error.
2964 	 * This is all done synchronously while the array is
2965 	 * frozen.
2966 	 */
2967 	bio = r10_bio->devs[slot].bio;
2968 	bio_put(bio);
2969 	r10_bio->devs[slot].bio = NULL;
2970 
2971 	if (mddev->ro)
2972 		r10_bio->devs[slot].bio = IO_BLOCKED;
2973 	else if (!test_bit(FailFast, &rdev->flags)) {
2974 		freeze_array(conf, 1);
2975 		fix_read_error(conf, mddev, r10_bio);
2976 		unfreeze_array(conf);
2977 	} else
2978 		md_error(mddev, rdev);
2979 
2980 	rdev_dec_pending(rdev, mddev);
2981 	allow_barrier(conf);
2982 	r10_bio->state = 0;
2983 	raid10_read_request(mddev, r10_bio->master_bio, r10_bio);
2984 }
2985 
handle_write_completed(struct r10conf * conf,struct r10bio * r10_bio)2986 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2987 {
2988 	/* Some sort of write request has finished and it
2989 	 * succeeded in writing where we thought there was a
2990 	 * bad block.  So forget the bad block.
2991 	 * Or possibly if failed and we need to record
2992 	 * a bad block.
2993 	 */
2994 	int m;
2995 	struct md_rdev *rdev;
2996 
2997 	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2998 	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2999 		for (m = 0; m < conf->copies; m++) {
3000 			int dev = r10_bio->devs[m].devnum;
3001 			rdev = conf->mirrors[dev].rdev;
3002 			if (r10_bio->devs[m].bio == NULL ||
3003 				r10_bio->devs[m].bio->bi_end_io == NULL)
3004 				continue;
3005 			if (!r10_bio->devs[m].bio->bi_status) {
3006 				rdev_clear_badblocks(
3007 					rdev,
3008 					r10_bio->devs[m].addr,
3009 					r10_bio->sectors, 0);
3010 			} else {
3011 				if (!rdev_set_badblocks(
3012 					    rdev,
3013 					    r10_bio->devs[m].addr,
3014 					    r10_bio->sectors, 0))
3015 					md_error(conf->mddev, rdev);
3016 			}
3017 			rdev = conf->mirrors[dev].replacement;
3018 			if (r10_bio->devs[m].repl_bio == NULL ||
3019 				r10_bio->devs[m].repl_bio->bi_end_io == NULL)
3020 				continue;
3021 
3022 			if (!r10_bio->devs[m].repl_bio->bi_status) {
3023 				rdev_clear_badblocks(
3024 					rdev,
3025 					r10_bio->devs[m].addr,
3026 					r10_bio->sectors, 0);
3027 			} else {
3028 				if (!rdev_set_badblocks(
3029 					    rdev,
3030 					    r10_bio->devs[m].addr,
3031 					    r10_bio->sectors, 0))
3032 					md_error(conf->mddev, rdev);
3033 			}
3034 		}
3035 		put_buf(r10_bio);
3036 	} else {
3037 		bool fail = false;
3038 		for (m = 0; m < conf->copies; m++) {
3039 			int dev = r10_bio->devs[m].devnum;
3040 			struct bio *bio = r10_bio->devs[m].bio;
3041 			rdev = conf->mirrors[dev].rdev;
3042 			if (bio == IO_MADE_GOOD) {
3043 				rdev_clear_badblocks(
3044 					rdev,
3045 					r10_bio->devs[m].addr,
3046 					r10_bio->sectors, 0);
3047 				rdev_dec_pending(rdev, conf->mddev);
3048 			} else if (bio != NULL && bio->bi_status) {
3049 				fail = true;
3050 				if (!narrow_write_error(r10_bio, m)) {
3051 					md_error(conf->mddev, rdev);
3052 					set_bit(R10BIO_Degraded,
3053 						&r10_bio->state);
3054 				}
3055 				rdev_dec_pending(rdev, conf->mddev);
3056 			}
3057 			bio = r10_bio->devs[m].repl_bio;
3058 			rdev = conf->mirrors[dev].replacement;
3059 			if (rdev && bio == IO_MADE_GOOD) {
3060 				rdev_clear_badblocks(
3061 					rdev,
3062 					r10_bio->devs[m].addr,
3063 					r10_bio->sectors, 0);
3064 				rdev_dec_pending(rdev, conf->mddev);
3065 			}
3066 		}
3067 		if (fail) {
3068 			spin_lock_irq(&conf->device_lock);
3069 			list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
3070 			conf->nr_queued++;
3071 			spin_unlock_irq(&conf->device_lock);
3072 			/*
3073 			 * In case freeze_array() is waiting for condition
3074 			 * nr_pending == nr_queued + extra to be true.
3075 			 */
3076 			wake_up(&conf->wait_barrier);
3077 			md_wakeup_thread(conf->mddev->thread);
3078 		} else {
3079 			if (test_bit(R10BIO_WriteError,
3080 				     &r10_bio->state))
3081 				close_write(r10_bio);
3082 			raid_end_bio_io(r10_bio);
3083 		}
3084 	}
3085 }
3086 
raid10d(struct md_thread * thread)3087 static void raid10d(struct md_thread *thread)
3088 {
3089 	struct mddev *mddev = thread->mddev;
3090 	struct r10bio *r10_bio;
3091 	unsigned long flags;
3092 	struct r10conf *conf = mddev->private;
3093 	struct list_head *head = &conf->retry_list;
3094 	struct blk_plug plug;
3095 
3096 	md_check_recovery(mddev);
3097 
3098 	if (!list_empty_careful(&conf->bio_end_io_list) &&
3099 	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
3100 		LIST_HEAD(tmp);
3101 		spin_lock_irqsave(&conf->device_lock, flags);
3102 		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
3103 			while (!list_empty(&conf->bio_end_io_list)) {
3104 				list_move(conf->bio_end_io_list.prev, &tmp);
3105 				conf->nr_queued--;
3106 			}
3107 		}
3108 		spin_unlock_irqrestore(&conf->device_lock, flags);
3109 		while (!list_empty(&tmp)) {
3110 			r10_bio = list_first_entry(&tmp, struct r10bio,
3111 						   retry_list);
3112 			list_del(&r10_bio->retry_list);
3113 			if (mddev->degraded)
3114 				set_bit(R10BIO_Degraded, &r10_bio->state);
3115 
3116 			if (test_bit(R10BIO_WriteError,
3117 				     &r10_bio->state))
3118 				close_write(r10_bio);
3119 			raid_end_bio_io(r10_bio);
3120 		}
3121 	}
3122 
3123 	blk_start_plug(&plug);
3124 	for (;;) {
3125 
3126 		flush_pending_writes(conf);
3127 
3128 		spin_lock_irqsave(&conf->device_lock, flags);
3129 		if (list_empty(head)) {
3130 			spin_unlock_irqrestore(&conf->device_lock, flags);
3131 			break;
3132 		}
3133 		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
3134 		list_del(head->prev);
3135 		conf->nr_queued--;
3136 		spin_unlock_irqrestore(&conf->device_lock, flags);
3137 
3138 		mddev = r10_bio->mddev;
3139 		conf = mddev->private;
3140 		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
3141 		    test_bit(R10BIO_WriteError, &r10_bio->state))
3142 			handle_write_completed(conf, r10_bio);
3143 		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
3144 			reshape_request_write(mddev, r10_bio);
3145 		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
3146 			sync_request_write(mddev, r10_bio);
3147 		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
3148 			recovery_request_write(mddev, r10_bio);
3149 		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
3150 			handle_read_error(mddev, r10_bio);
3151 		else
3152 			WARN_ON_ONCE(1);
3153 
3154 		cond_resched();
3155 		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
3156 			md_check_recovery(mddev);
3157 	}
3158 	blk_finish_plug(&plug);
3159 }
3160 
init_resync(struct r10conf * conf)3161 static int init_resync(struct r10conf *conf)
3162 {
3163 	int ret, buffs, i;
3164 
3165 	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
3166 	BUG_ON(mempool_initialized(&conf->r10buf_pool));
3167 	conf->have_replacement = 0;
3168 	for (i = 0; i < conf->geo.raid_disks; i++)
3169 		if (conf->mirrors[i].replacement)
3170 			conf->have_replacement = 1;
3171 	ret = mempool_init(&conf->r10buf_pool, buffs,
3172 			   r10buf_pool_alloc, r10buf_pool_free, conf);
3173 	if (ret)
3174 		return ret;
3175 	conf->next_resync = 0;
3176 	return 0;
3177 }
3178 
raid10_alloc_init_r10buf(struct r10conf * conf)3179 static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
3180 {
3181 	struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO);
3182 	struct rsync_pages *rp;
3183 	struct bio *bio;
3184 	int nalloc;
3185 	int i;
3186 
3187 	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
3188 	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
3189 		nalloc = conf->copies; /* resync */
3190 	else
3191 		nalloc = 2; /* recovery */
3192 
3193 	for (i = 0; i < nalloc; i++) {
3194 		bio = r10bio->devs[i].bio;
3195 		rp = bio->bi_private;
3196 		bio_reset(bio, NULL, 0);
3197 		bio->bi_private = rp;
3198 		bio = r10bio->devs[i].repl_bio;
3199 		if (bio) {
3200 			rp = bio->bi_private;
3201 			bio_reset(bio, NULL, 0);
3202 			bio->bi_private = rp;
3203 		}
3204 	}
3205 	return r10bio;
3206 }
3207 
3208 /*
3209  * Set cluster_sync_high since we need other nodes to add the
3210  * range [cluster_sync_low, cluster_sync_high] to suspend list.
3211  */
raid10_set_cluster_sync_high(struct r10conf * conf)3212 static void raid10_set_cluster_sync_high(struct r10conf *conf)
3213 {
3214 	sector_t window_size;
3215 	int extra_chunk, chunks;
3216 
3217 	/*
3218 	 * First, here we define "stripe" as a unit which across
3219 	 * all member devices one time, so we get chunks by use
3220 	 * raid_disks / near_copies. Otherwise, if near_copies is
3221 	 * close to raid_disks, then resync window could increases
3222 	 * linearly with the increase of raid_disks, which means
3223 	 * we will suspend a really large IO window while it is not
3224 	 * necessary. If raid_disks is not divisible by near_copies,
3225 	 * an extra chunk is needed to ensure the whole "stripe" is
3226 	 * covered.
3227 	 */
3228 
3229 	chunks = conf->geo.raid_disks / conf->geo.near_copies;
3230 	if (conf->geo.raid_disks % conf->geo.near_copies == 0)
3231 		extra_chunk = 0;
3232 	else
3233 		extra_chunk = 1;
3234 	window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;
3235 
3236 	/*
3237 	 * At least use a 32M window to align with raid1's resync window
3238 	 */
3239 	window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ?
3240 			CLUSTER_RESYNC_WINDOW_SECTORS : window_size;
3241 
3242 	conf->cluster_sync_high = conf->cluster_sync_low + window_size;
3243 }
3244 
3245 /*
3246  * perform a "sync" on one "block"
3247  *
3248  * We need to make sure that no normal I/O request - particularly write
3249  * requests - conflict with active sync requests.
3250  *
3251  * This is achieved by tracking pending requests and a 'barrier' concept
3252  * that can be installed to exclude normal IO requests.
3253  *
3254  * Resync and recovery are handled very differently.
3255  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
3256  *
3257  * For resync, we iterate over virtual addresses, read all copies,
3258  * and update if there are differences.  If only one copy is live,
3259  * skip it.
3260  * For recovery, we iterate over physical addresses, read a good
3261  * value for each non-in_sync drive, and over-write.
3262  *
3263  * So, for recovery we may have several outstanding complex requests for a
3264  * given address, one for each out-of-sync device.  We model this by allocating
3265  * a number of r10_bio structures, one for each out-of-sync device.
3266  * As we setup these structures, we collect all bio's together into a list
3267  * which we then process collectively to add pages, and then process again
3268  * to pass to submit_bio_noacct.
3269  *
3270  * The r10_bio structures are linked using a borrowed master_bio pointer.
3271  * This link is counted in ->remaining.  When the r10_bio that points to NULL
3272  * has its remaining count decremented to 0, the whole complex operation
3273  * is complete.
3274  *
3275  */
3276 
raid10_sync_request(struct mddev * mddev,sector_t sector_nr,int * skipped)3277 static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
3278 			     int *skipped)
3279 {
3280 	struct r10conf *conf = mddev->private;
3281 	struct r10bio *r10_bio;
3282 	struct bio *biolist = NULL, *bio;
3283 	sector_t max_sector, nr_sectors;
3284 	int i;
3285 	int max_sync;
3286 	sector_t sync_blocks;
3287 	sector_t sectors_skipped = 0;
3288 	int chunks_skipped = 0;
3289 	sector_t chunk_mask = conf->geo.chunk_mask;
3290 	int page_idx = 0;
3291 
3292 	if (!mempool_initialized(&conf->r10buf_pool))
3293 		if (init_resync(conf))
3294 			return 0;
3295 
3296 	/*
3297 	 * Allow skipping a full rebuild for incremental assembly
3298 	 * of a clean array, like RAID1 does.
3299 	 */
3300 	if (mddev->bitmap == NULL &&
3301 	    mddev->recovery_cp == MaxSector &&
3302 	    mddev->reshape_position == MaxSector &&
3303 	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
3304 	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
3305 	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
3306 	    conf->fullsync == 0) {
3307 		*skipped = 1;
3308 		return mddev->dev_sectors - sector_nr;
3309 	}
3310 
3311  skipped:
3312 	max_sector = mddev->dev_sectors;
3313 	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
3314 	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3315 		max_sector = mddev->resync_max_sectors;
3316 	if (sector_nr >= max_sector) {
3317 		conf->cluster_sync_low = 0;
3318 		conf->cluster_sync_high = 0;
3319 
3320 		/* If we aborted, we need to abort the
3321 		 * sync on the 'current' bitmap chucks (there can
3322 		 * be several when recovering multiple devices).
3323 		 * as we may have started syncing it but not finished.
3324 		 * We can find the current address in
3325 		 * mddev->curr_resync, but for recovery,
3326 		 * we need to convert that to several
3327 		 * virtual addresses.
3328 		 */
3329 		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
3330 			end_reshape(conf);
3331 			close_sync(conf);
3332 			return 0;
3333 		}
3334 
3335 		if (mddev->curr_resync < max_sector) { /* aborted */
3336 			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
3337 				md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
3338 						   &sync_blocks, 1);
3339 			else for (i = 0; i < conf->geo.raid_disks; i++) {
3340 				sector_t sect =
3341 					raid10_find_virt(conf, mddev->curr_resync, i);
3342 				md_bitmap_end_sync(mddev->bitmap, sect,
3343 						   &sync_blocks, 1);
3344 			}
3345 		} else {
3346 			/* completed sync */
3347 			if ((!mddev->bitmap || conf->fullsync)
3348 			    && conf->have_replacement
3349 			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3350 				/* Completed a full sync so the replacements
3351 				 * are now fully recovered.
3352 				 */
3353 				rcu_read_lock();
3354 				for (i = 0; i < conf->geo.raid_disks; i++) {
3355 					struct md_rdev *rdev =
3356 						rcu_dereference(conf->mirrors[i].replacement);
3357 					if (rdev)
3358 						rdev->recovery_offset = MaxSector;
3359 				}
3360 				rcu_read_unlock();
3361 			}
3362 			conf->fullsync = 0;
3363 		}
3364 		md_bitmap_close_sync(mddev->bitmap);
3365 		close_sync(conf);
3366 		*skipped = 1;
3367 		return sectors_skipped;
3368 	}
3369 
3370 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3371 		return reshape_request(mddev, sector_nr, skipped);
3372 
3373 	if (chunks_skipped >= conf->geo.raid_disks) {
3374 		/* if there has been nothing to do on any drive,
3375 		 * then there is nothing to do at all..
3376 		 */
3377 		*skipped = 1;
3378 		return (max_sector - sector_nr) + sectors_skipped;
3379 	}
3380 
3381 	if (max_sector > mddev->resync_max)
3382 		max_sector = mddev->resync_max; /* Don't do IO beyond here */
3383 
3384 	/* make sure whole request will fit in a chunk - if chunks
3385 	 * are meaningful
3386 	 */
3387 	if (conf->geo.near_copies < conf->geo.raid_disks &&
3388 	    max_sector > (sector_nr | chunk_mask))
3389 		max_sector = (sector_nr | chunk_mask) + 1;
3390 
3391 	/*
3392 	 * If there is non-resync activity waiting for a turn, then let it
3393 	 * though before starting on this new sync request.
3394 	 */
3395 	if (conf->nr_waiting)
3396 		schedule_timeout_uninterruptible(1);
3397 
3398 	/* Again, very different code for resync and recovery.
3399 	 * Both must result in an r10bio with a list of bios that
3400 	 * have bi_end_io, bi_sector, bi_bdev set,
3401 	 * and bi_private set to the r10bio.
3402 	 * For recovery, we may actually create several r10bios
3403 	 * with 2 bios in each, that correspond to the bios in the main one.
3404 	 * In this case, the subordinate r10bios link back through a
3405 	 * borrowed master_bio pointer, and the counter in the master
3406 	 * includes a ref from each subordinate.
3407 	 */
3408 	/* First, we decide what to do and set ->bi_end_io
3409 	 * To end_sync_read if we want to read, and
3410 	 * end_sync_write if we will want to write.
3411 	 */
3412 
3413 	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
3414 	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3415 		/* recovery... the complicated one */
3416 		int j;
3417 		r10_bio = NULL;
3418 
3419 		for (i = 0 ; i < conf->geo.raid_disks; i++) {
3420 			int still_degraded;
3421 			struct r10bio *rb2;
3422 			sector_t sect;
3423 			int must_sync;
3424 			int any_working;
3425 			int need_recover = 0;
3426 			int need_replace = 0;
3427 			struct raid10_info *mirror = &conf->mirrors[i];
3428 			struct md_rdev *mrdev, *mreplace;
3429 
3430 			rcu_read_lock();
3431 			mrdev = rcu_dereference(mirror->rdev);
3432 			mreplace = rcu_dereference(mirror->replacement);
3433 
3434 			if (mrdev != NULL &&
3435 			    !test_bit(Faulty, &mrdev->flags) &&
3436 			    !test_bit(In_sync, &mrdev->flags))
3437 				need_recover = 1;
3438 			if (mreplace != NULL &&
3439 			    !test_bit(Faulty, &mreplace->flags))
3440 				need_replace = 1;
3441 
3442 			if (!need_recover && !need_replace) {
3443 				rcu_read_unlock();
3444 				continue;
3445 			}
3446 
3447 			still_degraded = 0;
3448 			/* want to reconstruct this device */
3449 			rb2 = r10_bio;
3450 			sect = raid10_find_virt(conf, sector_nr, i);
3451 			if (sect >= mddev->resync_max_sectors) {
3452 				/* last stripe is not complete - don't
3453 				 * try to recover this sector.
3454 				 */
3455 				rcu_read_unlock();
3456 				continue;
3457 			}
3458 			if (mreplace && test_bit(Faulty, &mreplace->flags))
3459 				mreplace = NULL;
3460 			/* Unless we are doing a full sync, or a replacement
3461 			 * we only need to recover the block if it is set in
3462 			 * the bitmap
3463 			 */
3464 			must_sync = md_bitmap_start_sync(mddev->bitmap, sect,
3465 							 &sync_blocks, 1);
3466 			if (sync_blocks < max_sync)
3467 				max_sync = sync_blocks;
3468 			if (!must_sync &&
3469 			    mreplace == NULL &&
3470 			    !conf->fullsync) {
3471 				/* yep, skip the sync_blocks here, but don't assume
3472 				 * that there will never be anything to do here
3473 				 */
3474 				chunks_skipped = -1;
3475 				rcu_read_unlock();
3476 				continue;
3477 			}
3478 			atomic_inc(&mrdev->nr_pending);
3479 			if (mreplace)
3480 				atomic_inc(&mreplace->nr_pending);
3481 			rcu_read_unlock();
3482 
3483 			r10_bio = raid10_alloc_init_r10buf(conf);
3484 			r10_bio->state = 0;
3485 			raise_barrier(conf, rb2 != NULL);
3486 			atomic_set(&r10_bio->remaining, 0);
3487 
3488 			r10_bio->master_bio = (struct bio*)rb2;
3489 			if (rb2)
3490 				atomic_inc(&rb2->remaining);
3491 			r10_bio->mddev = mddev;
3492 			set_bit(R10BIO_IsRecover, &r10_bio->state);
3493 			r10_bio->sector = sect;
3494 
3495 			raid10_find_phys(conf, r10_bio);
3496 
3497 			/* Need to check if the array will still be
3498 			 * degraded
3499 			 */
3500 			rcu_read_lock();
3501 			for (j = 0; j < conf->geo.raid_disks; j++) {
3502 				struct md_rdev *rdev = rcu_dereference(
3503 					conf->mirrors[j].rdev);
3504 				if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3505 					still_degraded = 1;
3506 					break;
3507 				}
3508 			}
3509 
3510 			must_sync = md_bitmap_start_sync(mddev->bitmap, sect,
3511 							 &sync_blocks, still_degraded);
3512 
3513 			any_working = 0;
3514 			for (j=0; j<conf->copies;j++) {
3515 				int k;
3516 				int d = r10_bio->devs[j].devnum;
3517 				sector_t from_addr, to_addr;
3518 				struct md_rdev *rdev =
3519 					rcu_dereference(conf->mirrors[d].rdev);
3520 				sector_t sector, first_bad;
3521 				int bad_sectors;
3522 				if (!rdev ||
3523 				    !test_bit(In_sync, &rdev->flags))
3524 					continue;
3525 				/* This is where we read from */
3526 				any_working = 1;
3527 				sector = r10_bio->devs[j].addr;
3528 
3529 				if (is_badblock(rdev, sector, max_sync,
3530 						&first_bad, &bad_sectors)) {
3531 					if (first_bad > sector)
3532 						max_sync = first_bad - sector;
3533 					else {
3534 						bad_sectors -= (sector
3535 								- first_bad);
3536 						if (max_sync > bad_sectors)
3537 							max_sync = bad_sectors;
3538 						continue;
3539 					}
3540 				}
3541 				bio = r10_bio->devs[0].bio;
3542 				bio->bi_next = biolist;
3543 				biolist = bio;
3544 				bio->bi_end_io = end_sync_read;
3545 				bio_set_op_attrs(bio, REQ_OP_READ, 0);
3546 				if (test_bit(FailFast, &rdev->flags))
3547 					bio->bi_opf |= MD_FAILFAST;
3548 				from_addr = r10_bio->devs[j].addr;
3549 				bio->bi_iter.bi_sector = from_addr +
3550 					rdev->data_offset;
3551 				bio_set_dev(bio, rdev->bdev);
3552 				atomic_inc(&rdev->nr_pending);
3553 				/* and we write to 'i' (if not in_sync) */
3554 
3555 				for (k=0; k<conf->copies; k++)
3556 					if (r10_bio->devs[k].devnum == i)
3557 						break;
3558 				BUG_ON(k == conf->copies);
3559 				to_addr = r10_bio->devs[k].addr;
3560 				r10_bio->devs[0].devnum = d;
3561 				r10_bio->devs[0].addr = from_addr;
3562 				r10_bio->devs[1].devnum = i;
3563 				r10_bio->devs[1].addr = to_addr;
3564 
3565 				if (need_recover) {
3566 					bio = r10_bio->devs[1].bio;
3567 					bio->bi_next = biolist;
3568 					biolist = bio;
3569 					bio->bi_end_io = end_sync_write;
3570 					bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3571 					bio->bi_iter.bi_sector = to_addr
3572 						+ mrdev->data_offset;
3573 					bio_set_dev(bio, mrdev->bdev);
3574 					atomic_inc(&r10_bio->remaining);
3575 				} else
3576 					r10_bio->devs[1].bio->bi_end_io = NULL;
3577 
3578 				/* and maybe write to replacement */
3579 				bio = r10_bio->devs[1].repl_bio;
3580 				if (bio)
3581 					bio->bi_end_io = NULL;
3582 				/* Note: if need_replace, then bio
3583 				 * cannot be NULL as r10buf_pool_alloc will
3584 				 * have allocated it.
3585 				 */
3586 				if (!need_replace)
3587 					break;
3588 				bio->bi_next = biolist;
3589 				biolist = bio;
3590 				bio->bi_end_io = end_sync_write;
3591 				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3592 				bio->bi_iter.bi_sector = to_addr +
3593 					mreplace->data_offset;
3594 				bio_set_dev(bio, mreplace->bdev);
3595 				atomic_inc(&r10_bio->remaining);
3596 				break;
3597 			}
3598 			rcu_read_unlock();
3599 			if (j == conf->copies) {
3600 				/* Cannot recover, so abort the recovery or
3601 				 * record a bad block */
3602 				if (any_working) {
3603 					/* problem is that there are bad blocks
3604 					 * on other device(s)
3605 					 */
3606 					int k;
3607 					for (k = 0; k < conf->copies; k++)
3608 						if (r10_bio->devs[k].devnum == i)
3609 							break;
3610 					if (!test_bit(In_sync,
3611 						      &mrdev->flags)
3612 					    && !rdev_set_badblocks(
3613 						    mrdev,
3614 						    r10_bio->devs[k].addr,
3615 						    max_sync, 0))
3616 						any_working = 0;
3617 					if (mreplace &&
3618 					    !rdev_set_badblocks(
3619 						    mreplace,
3620 						    r10_bio->devs[k].addr,
3621 						    max_sync, 0))
3622 						any_working = 0;
3623 				}
3624 				if (!any_working)  {
3625 					if (!test_and_set_bit(MD_RECOVERY_INTR,
3626 							      &mddev->recovery))
3627 						pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3628 						       mdname(mddev));
3629 					mirror->recovery_disabled
3630 						= mddev->recovery_disabled;
3631 				}
3632 				put_buf(r10_bio);
3633 				if (rb2)
3634 					atomic_dec(&rb2->remaining);
3635 				r10_bio = rb2;
3636 				rdev_dec_pending(mrdev, mddev);
3637 				if (mreplace)
3638 					rdev_dec_pending(mreplace, mddev);
3639 				break;
3640 			}
3641 			rdev_dec_pending(mrdev, mddev);
3642 			if (mreplace)
3643 				rdev_dec_pending(mreplace, mddev);
3644 			if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
3645 				/* Only want this if there is elsewhere to
3646 				 * read from. 'j' is currently the first
3647 				 * readable copy.
3648 				 */
3649 				int targets = 1;
3650 				for (; j < conf->copies; j++) {
3651 					int d = r10_bio->devs[j].devnum;
3652 					if (conf->mirrors[d].rdev &&
3653 					    test_bit(In_sync,
3654 						      &conf->mirrors[d].rdev->flags))
3655 						targets++;
3656 				}
3657 				if (targets == 1)
3658 					r10_bio->devs[0].bio->bi_opf
3659 						&= ~MD_FAILFAST;
3660 			}
3661 		}
3662 		if (biolist == NULL) {
3663 			while (r10_bio) {
3664 				struct r10bio *rb2 = r10_bio;
3665 				r10_bio = (struct r10bio*) rb2->master_bio;
3666 				rb2->master_bio = NULL;
3667 				put_buf(rb2);
3668 			}
3669 			goto giveup;
3670 		}
3671 	} else {
3672 		/* resync. Schedule a read for every block at this virt offset */
3673 		int count = 0;
3674 
3675 		/*
3676 		 * Since curr_resync_completed could probably not update in
3677 		 * time, and we will set cluster_sync_low based on it.
3678 		 * Let's check against "sector_nr + 2 * RESYNC_SECTORS" for
3679 		 * safety reason, which ensures curr_resync_completed is
3680 		 * updated in bitmap_cond_end_sync.
3681 		 */
3682 		md_bitmap_cond_end_sync(mddev->bitmap, sector_nr,
3683 					mddev_is_clustered(mddev) &&
3684 					(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
3685 
3686 		if (!md_bitmap_start_sync(mddev->bitmap, sector_nr,
3687 					  &sync_blocks, mddev->degraded) &&
3688 		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3689 						 &mddev->recovery)) {
3690 			/* We can skip this block */
3691 			*skipped = 1;
3692 			return sync_blocks + sectors_skipped;
3693 		}
3694 		if (sync_blocks < max_sync)
3695 			max_sync = sync_blocks;
3696 		r10_bio = raid10_alloc_init_r10buf(conf);
3697 		r10_bio->state = 0;
3698 
3699 		r10_bio->mddev = mddev;
3700 		atomic_set(&r10_bio->remaining, 0);
3701 		raise_barrier(conf, 0);
3702 		conf->next_resync = sector_nr;
3703 
3704 		r10_bio->master_bio = NULL;
3705 		r10_bio->sector = sector_nr;
3706 		set_bit(R10BIO_IsSync, &r10_bio->state);
3707 		raid10_find_phys(conf, r10_bio);
3708 		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3709 
3710 		for (i = 0; i < conf->copies; i++) {
3711 			int d = r10_bio->devs[i].devnum;
3712 			sector_t first_bad, sector;
3713 			int bad_sectors;
3714 			struct md_rdev *rdev;
3715 
3716 			if (r10_bio->devs[i].repl_bio)
3717 				r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3718 
3719 			bio = r10_bio->devs[i].bio;
3720 			bio->bi_status = BLK_STS_IOERR;
3721 			rcu_read_lock();
3722 			rdev = rcu_dereference(conf->mirrors[d].rdev);
3723 			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3724 				rcu_read_unlock();
3725 				continue;
3726 			}
3727 			sector = r10_bio->devs[i].addr;
3728 			if (is_badblock(rdev, sector, max_sync,
3729 					&first_bad, &bad_sectors)) {
3730 				if (first_bad > sector)
3731 					max_sync = first_bad - sector;
3732 				else {
3733 					bad_sectors -= (sector - first_bad);
3734 					if (max_sync > bad_sectors)
3735 						max_sync = bad_sectors;
3736 					rcu_read_unlock();
3737 					continue;
3738 				}
3739 			}
3740 			atomic_inc(&rdev->nr_pending);
3741 			atomic_inc(&r10_bio->remaining);
3742 			bio->bi_next = biolist;
3743 			biolist = bio;
3744 			bio->bi_end_io = end_sync_read;
3745 			bio_set_op_attrs(bio, REQ_OP_READ, 0);
3746 			if (test_bit(FailFast, &rdev->flags))
3747 				bio->bi_opf |= MD_FAILFAST;
3748 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
3749 			bio_set_dev(bio, rdev->bdev);
3750 			count++;
3751 
3752 			rdev = rcu_dereference(conf->mirrors[d].replacement);
3753 			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3754 				rcu_read_unlock();
3755 				continue;
3756 			}
3757 			atomic_inc(&rdev->nr_pending);
3758 
3759 			/* Need to set up for writing to the replacement */
3760 			bio = r10_bio->devs[i].repl_bio;
3761 			bio->bi_status = BLK_STS_IOERR;
3762 
3763 			sector = r10_bio->devs[i].addr;
3764 			bio->bi_next = biolist;
3765 			biolist = bio;
3766 			bio->bi_end_io = end_sync_write;
3767 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3768 			if (test_bit(FailFast, &rdev->flags))
3769 				bio->bi_opf |= MD_FAILFAST;
3770 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
3771 			bio_set_dev(bio, rdev->bdev);
3772 			count++;
3773 			rcu_read_unlock();
3774 		}
3775 
3776 		if (count < 2) {
3777 			for (i=0; i<conf->copies; i++) {
3778 				int d = r10_bio->devs[i].devnum;
3779 				if (r10_bio->devs[i].bio->bi_end_io)
3780 					rdev_dec_pending(conf->mirrors[d].rdev,
3781 							 mddev);
3782 				if (r10_bio->devs[i].repl_bio &&
3783 				    r10_bio->devs[i].repl_bio->bi_end_io)
3784 					rdev_dec_pending(
3785 						conf->mirrors[d].replacement,
3786 						mddev);
3787 			}
3788 			put_buf(r10_bio);
3789 			biolist = NULL;
3790 			goto giveup;
3791 		}
3792 	}
3793 
3794 	nr_sectors = 0;
3795 	if (sector_nr + max_sync < max_sector)
3796 		max_sector = sector_nr + max_sync;
3797 	do {
3798 		struct page *page;
3799 		int len = PAGE_SIZE;
3800 		if (sector_nr + (len>>9) > max_sector)
3801 			len = (max_sector - sector_nr) << 9;
3802 		if (len == 0)
3803 			break;
3804 		for (bio= biolist ; bio ; bio=bio->bi_next) {
3805 			struct resync_pages *rp = get_resync_pages(bio);
3806 			page = resync_fetch_page(rp, page_idx);
3807 			/*
3808 			 * won't fail because the vec table is big enough
3809 			 * to hold all these pages
3810 			 */
3811 			bio_add_page(bio, page, len, 0);
3812 		}
3813 		nr_sectors += len>>9;
3814 		sector_nr += len>>9;
3815 	} while (++page_idx < RESYNC_PAGES);
3816 	r10_bio->sectors = nr_sectors;
3817 
3818 	if (mddev_is_clustered(mddev) &&
3819 	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3820 		/* It is resync not recovery */
3821 		if (conf->cluster_sync_high < sector_nr + nr_sectors) {
3822 			conf->cluster_sync_low = mddev->curr_resync_completed;
3823 			raid10_set_cluster_sync_high(conf);
3824 			/* Send resync message */
3825 			md_cluster_ops->resync_info_update(mddev,
3826 						conf->cluster_sync_low,
3827 						conf->cluster_sync_high);
3828 		}
3829 	} else if (mddev_is_clustered(mddev)) {
3830 		/* This is recovery not resync */
3831 		sector_t sect_va1, sect_va2;
3832 		bool broadcast_msg = false;
3833 
3834 		for (i = 0; i < conf->geo.raid_disks; i++) {
3835 			/*
3836 			 * sector_nr is a device address for recovery, so we
3837 			 * need translate it to array address before compare
3838 			 * with cluster_sync_high.
3839 			 */
3840 			sect_va1 = raid10_find_virt(conf, sector_nr, i);
3841 
3842 			if (conf->cluster_sync_high < sect_va1 + nr_sectors) {
3843 				broadcast_msg = true;
3844 				/*
3845 				 * curr_resync_completed is similar as
3846 				 * sector_nr, so make the translation too.
3847 				 */
3848 				sect_va2 = raid10_find_virt(conf,
3849 					mddev->curr_resync_completed, i);
3850 
3851 				if (conf->cluster_sync_low == 0 ||
3852 				    conf->cluster_sync_low > sect_va2)
3853 					conf->cluster_sync_low = sect_va2;
3854 			}
3855 		}
3856 		if (broadcast_msg) {
3857 			raid10_set_cluster_sync_high(conf);
3858 			md_cluster_ops->resync_info_update(mddev,
3859 						conf->cluster_sync_low,
3860 						conf->cluster_sync_high);
3861 		}
3862 	}
3863 
3864 	while (biolist) {
3865 		bio = biolist;
3866 		biolist = biolist->bi_next;
3867 
3868 		bio->bi_next = NULL;
3869 		r10_bio = get_resync_r10bio(bio);
3870 		r10_bio->sectors = nr_sectors;
3871 
3872 		if (bio->bi_end_io == end_sync_read) {
3873 			md_sync_acct_bio(bio, nr_sectors);
3874 			bio->bi_status = 0;
3875 			submit_bio_noacct(bio);
3876 		}
3877 	}
3878 
3879 	if (sectors_skipped)
3880 		/* pretend they weren't skipped, it makes
3881 		 * no important difference in this case
3882 		 */
3883 		md_done_sync(mddev, sectors_skipped, 1);
3884 
3885 	return sectors_skipped + nr_sectors;
3886  giveup:
3887 	/* There is nowhere to write, so all non-sync
3888 	 * drives must be failed or in resync, all drives
3889 	 * have a bad block, so try the next chunk...
3890 	 */
3891 	if (sector_nr + max_sync < max_sector)
3892 		max_sector = sector_nr + max_sync;
3893 
3894 	sectors_skipped += (max_sector - sector_nr);
3895 	chunks_skipped ++;
3896 	sector_nr = max_sector;
3897 	goto skipped;
3898 }
3899 
3900 static sector_t
raid10_size(struct mddev * mddev,sector_t sectors,int raid_disks)3901 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3902 {
3903 	sector_t size;
3904 	struct r10conf *conf = mddev->private;
3905 
3906 	if (!raid_disks)
3907 		raid_disks = min(conf->geo.raid_disks,
3908 				 conf->prev.raid_disks);
3909 	if (!sectors)
3910 		sectors = conf->dev_sectors;
3911 
3912 	size = sectors >> conf->geo.chunk_shift;
3913 	sector_div(size, conf->geo.far_copies);
3914 	size = size * raid_disks;
3915 	sector_div(size, conf->geo.near_copies);
3916 
3917 	return size << conf->geo.chunk_shift;
3918 }
3919 
calc_sectors(struct r10conf * conf,sector_t size)3920 static void calc_sectors(struct r10conf *conf, sector_t size)
3921 {
3922 	/* Calculate the number of sectors-per-device that will
3923 	 * actually be used, and set conf->dev_sectors and
3924 	 * conf->stride
3925 	 */
3926 
3927 	size = size >> conf->geo.chunk_shift;
3928 	sector_div(size, conf->geo.far_copies);
3929 	size = size * conf->geo.raid_disks;
3930 	sector_div(size, conf->geo.near_copies);
3931 	/* 'size' is now the number of chunks in the array */
3932 	/* calculate "used chunks per device" */
3933 	size = size * conf->copies;
3934 
3935 	/* We need to round up when dividing by raid_disks to
3936 	 * get the stride size.
3937 	 */
3938 	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3939 
3940 	conf->dev_sectors = size << conf->geo.chunk_shift;
3941 
3942 	if (conf->geo.far_offset)
3943 		conf->geo.stride = 1 << conf->geo.chunk_shift;
3944 	else {
3945 		sector_div(size, conf->geo.far_copies);
3946 		conf->geo.stride = size << conf->geo.chunk_shift;
3947 	}
3948 }
3949 
3950 enum geo_type {geo_new, geo_old, geo_start};
setup_geo(struct geom * geo,struct mddev * mddev,enum geo_type new)3951 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3952 {
3953 	int nc, fc, fo;
3954 	int layout, chunk, disks;
3955 	switch (new) {
3956 	case geo_old:
3957 		layout = mddev->layout;
3958 		chunk = mddev->chunk_sectors;
3959 		disks = mddev->raid_disks - mddev->delta_disks;
3960 		break;
3961 	case geo_new:
3962 		layout = mddev->new_layout;
3963 		chunk = mddev->new_chunk_sectors;
3964 		disks = mddev->raid_disks;
3965 		break;
3966 	default: /* avoid 'may be unused' warnings */
3967 	case geo_start: /* new when starting reshape - raid_disks not
3968 			 * updated yet. */
3969 		layout = mddev->new_layout;
3970 		chunk = mddev->new_chunk_sectors;
3971 		disks = mddev->raid_disks + mddev->delta_disks;
3972 		break;
3973 	}
3974 	if (layout >> 19)
3975 		return -1;
3976 	if (chunk < (PAGE_SIZE >> 9) ||
3977 	    !is_power_of_2(chunk))
3978 		return -2;
3979 	nc = layout & 255;
3980 	fc = (layout >> 8) & 255;
3981 	fo = layout & (1<<16);
3982 	geo->raid_disks = disks;
3983 	geo->near_copies = nc;
3984 	geo->far_copies = fc;
3985 	geo->far_offset = fo;
3986 	switch (layout >> 17) {
3987 	case 0:	/* original layout.  simple but not always optimal */
3988 		geo->far_set_size = disks;
3989 		break;
3990 	case 1: /* "improved" layout which was buggy.  Hopefully no-one is
3991 		 * actually using this, but leave code here just in case.*/
3992 		geo->far_set_size = disks/fc;
3993 		WARN(geo->far_set_size < fc,
3994 		     "This RAID10 layout does not provide data safety - please backup and create new array\n");
3995 		break;
3996 	case 2: /* "improved" layout fixed to match documentation */
3997 		geo->far_set_size = fc * nc;
3998 		break;
3999 	default: /* Not a valid layout */
4000 		return -1;
4001 	}
4002 	geo->chunk_mask = chunk - 1;
4003 	geo->chunk_shift = ffz(~chunk);
4004 	return nc*fc;
4005 }
4006 
setup_conf(struct mddev * mddev)4007 static struct r10conf *setup_conf(struct mddev *mddev)
4008 {
4009 	struct r10conf *conf = NULL;
4010 	int err = -EINVAL;
4011 	struct geom geo;
4012 	int copies;
4013 
4014 	copies = setup_geo(&geo, mddev, geo_new);
4015 
4016 	if (copies == -2) {
4017 		pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
4018 			mdname(mddev), PAGE_SIZE);
4019 		goto out;
4020 	}
4021 
4022 	if (copies < 2 || copies > mddev->raid_disks) {
4023 		pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
4024 			mdname(mddev), mddev->new_layout);
4025 		goto out;
4026 	}
4027 
4028 	err = -ENOMEM;
4029 	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
4030 	if (!conf)
4031 		goto out;
4032 
4033 	/* FIXME calc properly */
4034 	conf->mirrors = kcalloc(mddev->raid_disks + max(0, -mddev->delta_disks),
4035 				sizeof(struct raid10_info),
4036 				GFP_KERNEL);
4037 	if (!conf->mirrors)
4038 		goto out;
4039 
4040 	conf->tmppage = alloc_page(GFP_KERNEL);
4041 	if (!conf->tmppage)
4042 		goto out;
4043 
4044 	conf->geo = geo;
4045 	conf->copies = copies;
4046 	err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc,
4047 			   rbio_pool_free, conf);
4048 	if (err)
4049 		goto out;
4050 
4051 	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
4052 	if (err)
4053 		goto out;
4054 
4055 	calc_sectors(conf, mddev->dev_sectors);
4056 	if (mddev->reshape_position == MaxSector) {
4057 		conf->prev = conf->geo;
4058 		conf->reshape_progress = MaxSector;
4059 	} else {
4060 		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
4061 			err = -EINVAL;
4062 			goto out;
4063 		}
4064 		conf->reshape_progress = mddev->reshape_position;
4065 		if (conf->prev.far_offset)
4066 			conf->prev.stride = 1 << conf->prev.chunk_shift;
4067 		else
4068 			/* far_copies must be 1 */
4069 			conf->prev.stride = conf->dev_sectors;
4070 	}
4071 	conf->reshape_safe = conf->reshape_progress;
4072 	spin_lock_init(&conf->device_lock);
4073 	INIT_LIST_HEAD(&conf->retry_list);
4074 	INIT_LIST_HEAD(&conf->bio_end_io_list);
4075 
4076 	seqlock_init(&conf->resync_lock);
4077 	init_waitqueue_head(&conf->wait_barrier);
4078 	atomic_set(&conf->nr_pending, 0);
4079 
4080 	err = -ENOMEM;
4081 	conf->thread = md_register_thread(raid10d, mddev, "raid10");
4082 	if (!conf->thread)
4083 		goto out;
4084 
4085 	conf->mddev = mddev;
4086 	return conf;
4087 
4088  out:
4089 	if (conf) {
4090 		mempool_exit(&conf->r10bio_pool);
4091 		kfree(conf->mirrors);
4092 		safe_put_page(conf->tmppage);
4093 		bioset_exit(&conf->bio_split);
4094 		kfree(conf);
4095 	}
4096 	return ERR_PTR(err);
4097 }
4098 
raid10_set_io_opt(struct r10conf * conf)4099 static void raid10_set_io_opt(struct r10conf *conf)
4100 {
4101 	int raid_disks = conf->geo.raid_disks;
4102 
4103 	if (!(conf->geo.raid_disks % conf->geo.near_copies))
4104 		raid_disks /= conf->geo.near_copies;
4105 	blk_queue_io_opt(conf->mddev->queue, (conf->mddev->chunk_sectors << 9) *
4106 			 raid_disks);
4107 }
4108 
raid10_run(struct mddev * mddev)4109 static int raid10_run(struct mddev *mddev)
4110 {
4111 	struct r10conf *conf;
4112 	int i, disk_idx;
4113 	struct raid10_info *disk;
4114 	struct md_rdev *rdev;
4115 	sector_t size;
4116 	sector_t min_offset_diff = 0;
4117 	int first = 1;
4118 
4119 	if (mddev_init_writes_pending(mddev) < 0)
4120 		return -ENOMEM;
4121 
4122 	if (mddev->private == NULL) {
4123 		conf = setup_conf(mddev);
4124 		if (IS_ERR(conf))
4125 			return PTR_ERR(conf);
4126 		mddev->private = conf;
4127 	}
4128 	conf = mddev->private;
4129 	if (!conf)
4130 		goto out;
4131 
4132 	if (mddev_is_clustered(conf->mddev)) {
4133 		int fc, fo;
4134 
4135 		fc = (mddev->layout >> 8) & 255;
4136 		fo = mddev->layout & (1<<16);
4137 		if (fc > 1 || fo > 0) {
4138 			pr_err("only near layout is supported by clustered"
4139 				" raid10\n");
4140 			goto out_free_conf;
4141 		}
4142 	}
4143 
4144 	mddev->thread = conf->thread;
4145 	conf->thread = NULL;
4146 
4147 	if (mddev->queue) {
4148 		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
4149 		blk_queue_io_min(mddev->queue, mddev->chunk_sectors << 9);
4150 		raid10_set_io_opt(conf);
4151 	}
4152 
4153 	rdev_for_each(rdev, mddev) {
4154 		long long diff;
4155 
4156 		disk_idx = rdev->raid_disk;
4157 		if (disk_idx < 0)
4158 			continue;
4159 		if (disk_idx >= conf->geo.raid_disks &&
4160 		    disk_idx >= conf->prev.raid_disks)
4161 			continue;
4162 		disk = conf->mirrors + disk_idx;
4163 
4164 		if (test_bit(Replacement, &rdev->flags)) {
4165 			if (disk->replacement)
4166 				goto out_free_conf;
4167 			disk->replacement = rdev;
4168 		} else {
4169 			if (disk->rdev)
4170 				goto out_free_conf;
4171 			disk->rdev = rdev;
4172 		}
4173 		diff = (rdev->new_data_offset - rdev->data_offset);
4174 		if (!mddev->reshape_backwards)
4175 			diff = -diff;
4176 		if (diff < 0)
4177 			diff = 0;
4178 		if (first || diff < min_offset_diff)
4179 			min_offset_diff = diff;
4180 
4181 		if (mddev->gendisk)
4182 			disk_stack_limits(mddev->gendisk, rdev->bdev,
4183 					  rdev->data_offset << 9);
4184 
4185 		disk->head_position = 0;
4186 		first = 0;
4187 	}
4188 
4189 	/* need to check that every block has at least one working mirror */
4190 	if (!enough(conf, -1)) {
4191 		pr_err("md/raid10:%s: not enough operational mirrors.\n",
4192 		       mdname(mddev));
4193 		goto out_free_conf;
4194 	}
4195 
4196 	if (conf->reshape_progress != MaxSector) {
4197 		/* must ensure that shape change is supported */
4198 		if (conf->geo.far_copies != 1 &&
4199 		    conf->geo.far_offset == 0)
4200 			goto out_free_conf;
4201 		if (conf->prev.far_copies != 1 &&
4202 		    conf->prev.far_offset == 0)
4203 			goto out_free_conf;
4204 	}
4205 
4206 	mddev->degraded = 0;
4207 	for (i = 0;
4208 	     i < conf->geo.raid_disks
4209 		     || i < conf->prev.raid_disks;
4210 	     i++) {
4211 
4212 		disk = conf->mirrors + i;
4213 
4214 		if (!disk->rdev && disk->replacement) {
4215 			/* The replacement is all we have - use it */
4216 			disk->rdev = disk->replacement;
4217 			disk->replacement = NULL;
4218 			clear_bit(Replacement, &disk->rdev->flags);
4219 		}
4220 
4221 		if (!disk->rdev ||
4222 		    !test_bit(In_sync, &disk->rdev->flags)) {
4223 			disk->head_position = 0;
4224 			mddev->degraded++;
4225 			if (disk->rdev &&
4226 			    disk->rdev->saved_raid_disk < 0)
4227 				conf->fullsync = 1;
4228 		}
4229 
4230 		if (disk->replacement &&
4231 		    !test_bit(In_sync, &disk->replacement->flags) &&
4232 		    disk->replacement->saved_raid_disk < 0) {
4233 			conf->fullsync = 1;
4234 		}
4235 
4236 		disk->recovery_disabled = mddev->recovery_disabled - 1;
4237 	}
4238 
4239 	if (mddev->recovery_cp != MaxSector)
4240 		pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
4241 			  mdname(mddev));
4242 	pr_info("md/raid10:%s: active with %d out of %d devices\n",
4243 		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
4244 		conf->geo.raid_disks);
4245 	/*
4246 	 * Ok, everything is just fine now
4247 	 */
4248 	mddev->dev_sectors = conf->dev_sectors;
4249 	size = raid10_size(mddev, 0, 0);
4250 	md_set_array_sectors(mddev, size);
4251 	mddev->resync_max_sectors = size;
4252 	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
4253 
4254 	if (md_integrity_register(mddev))
4255 		goto out_free_conf;
4256 
4257 	if (conf->reshape_progress != MaxSector) {
4258 		unsigned long before_length, after_length;
4259 
4260 		before_length = ((1 << conf->prev.chunk_shift) *
4261 				 conf->prev.far_copies);
4262 		after_length = ((1 << conf->geo.chunk_shift) *
4263 				conf->geo.far_copies);
4264 
4265 		if (max(before_length, after_length) > min_offset_diff) {
4266 			/* This cannot work */
4267 			pr_warn("md/raid10: offset difference not enough to continue reshape\n");
4268 			goto out_free_conf;
4269 		}
4270 		conf->offset_diff = min_offset_diff;
4271 
4272 		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4273 		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4274 		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4275 		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4276 		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4277 							"reshape");
4278 		if (!mddev->sync_thread)
4279 			goto out_free_conf;
4280 	}
4281 
4282 	return 0;
4283 
4284 out_free_conf:
4285 	md_unregister_thread(&mddev->thread);
4286 	mempool_exit(&conf->r10bio_pool);
4287 	safe_put_page(conf->tmppage);
4288 	kfree(conf->mirrors);
4289 	kfree(conf);
4290 	mddev->private = NULL;
4291 out:
4292 	return -EIO;
4293 }
4294 
raid10_free(struct mddev * mddev,void * priv)4295 static void raid10_free(struct mddev *mddev, void *priv)
4296 {
4297 	struct r10conf *conf = priv;
4298 
4299 	mempool_exit(&conf->r10bio_pool);
4300 	safe_put_page(conf->tmppage);
4301 	kfree(conf->mirrors);
4302 	kfree(conf->mirrors_old);
4303 	kfree(conf->mirrors_new);
4304 	bioset_exit(&conf->bio_split);
4305 	kfree(conf);
4306 }
4307 
raid10_quiesce(struct mddev * mddev,int quiesce)4308 static void raid10_quiesce(struct mddev *mddev, int quiesce)
4309 {
4310 	struct r10conf *conf = mddev->private;
4311 
4312 	if (quiesce)
4313 		raise_barrier(conf, 0);
4314 	else
4315 		lower_barrier(conf);
4316 }
4317 
raid10_resize(struct mddev * mddev,sector_t sectors)4318 static int raid10_resize(struct mddev *mddev, sector_t sectors)
4319 {
4320 	/* Resize of 'far' arrays is not supported.
4321 	 * For 'near' and 'offset' arrays we can set the
4322 	 * number of sectors used to be an appropriate multiple
4323 	 * of the chunk size.
4324 	 * For 'offset', this is far_copies*chunksize.
4325 	 * For 'near' the multiplier is the LCM of
4326 	 * near_copies and raid_disks.
4327 	 * So if far_copies > 1 && !far_offset, fail.
4328 	 * Else find LCM(raid_disks, near_copy)*far_copies and
4329 	 * multiply by chunk_size.  Then round to this number.
4330 	 * This is mostly done by raid10_size()
4331 	 */
4332 	struct r10conf *conf = mddev->private;
4333 	sector_t oldsize, size;
4334 
4335 	if (mddev->reshape_position != MaxSector)
4336 		return -EBUSY;
4337 
4338 	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
4339 		return -EINVAL;
4340 
4341 	oldsize = raid10_size(mddev, 0, 0);
4342 	size = raid10_size(mddev, sectors, 0);
4343 	if (mddev->external_size &&
4344 	    mddev->array_sectors > size)
4345 		return -EINVAL;
4346 	if (mddev->bitmap) {
4347 		int ret = md_bitmap_resize(mddev->bitmap, size, 0, 0);
4348 		if (ret)
4349 			return ret;
4350 	}
4351 	md_set_array_sectors(mddev, size);
4352 	if (sectors > mddev->dev_sectors &&
4353 	    mddev->recovery_cp > oldsize) {
4354 		mddev->recovery_cp = oldsize;
4355 		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
4356 	}
4357 	calc_sectors(conf, sectors);
4358 	mddev->dev_sectors = conf->dev_sectors;
4359 	mddev->resync_max_sectors = size;
4360 	return 0;
4361 }
4362 
raid10_takeover_raid0(struct mddev * mddev,sector_t size,int devs)4363 static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
4364 {
4365 	struct md_rdev *rdev;
4366 	struct r10conf *conf;
4367 
4368 	if (mddev->degraded > 0) {
4369 		pr_warn("md/raid10:%s: Error: degraded raid0!\n",
4370 			mdname(mddev));
4371 		return ERR_PTR(-EINVAL);
4372 	}
4373 	sector_div(size, devs);
4374 
4375 	/* Set new parameters */
4376 	mddev->new_level = 10;
4377 	/* new layout: far_copies = 1, near_copies = 2 */
4378 	mddev->new_layout = (1<<8) + 2;
4379 	mddev->new_chunk_sectors = mddev->chunk_sectors;
4380 	mddev->delta_disks = mddev->raid_disks;
4381 	mddev->raid_disks *= 2;
4382 	/* make sure it will be not marked as dirty */
4383 	mddev->recovery_cp = MaxSector;
4384 	mddev->dev_sectors = size;
4385 
4386 	conf = setup_conf(mddev);
4387 	if (!IS_ERR(conf)) {
4388 		rdev_for_each(rdev, mddev)
4389 			if (rdev->raid_disk >= 0) {
4390 				rdev->new_raid_disk = rdev->raid_disk * 2;
4391 				rdev->sectors = size;
4392 			}
4393 		WRITE_ONCE(conf->barrier, 1);
4394 	}
4395 
4396 	return conf;
4397 }
4398 
raid10_takeover(struct mddev * mddev)4399 static void *raid10_takeover(struct mddev *mddev)
4400 {
4401 	struct r0conf *raid0_conf;
4402 
4403 	/* raid10 can take over:
4404 	 *  raid0 - providing it has only two drives
4405 	 */
4406 	if (mddev->level == 0) {
4407 		/* for raid0 takeover only one zone is supported */
4408 		raid0_conf = mddev->private;
4409 		if (raid0_conf->nr_strip_zones > 1) {
4410 			pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
4411 				mdname(mddev));
4412 			return ERR_PTR(-EINVAL);
4413 		}
4414 		return raid10_takeover_raid0(mddev,
4415 			raid0_conf->strip_zone->zone_end,
4416 			raid0_conf->strip_zone->nb_dev);
4417 	}
4418 	return ERR_PTR(-EINVAL);
4419 }
4420 
raid10_check_reshape(struct mddev * mddev)4421 static int raid10_check_reshape(struct mddev *mddev)
4422 {
4423 	/* Called when there is a request to change
4424 	 * - layout (to ->new_layout)
4425 	 * - chunk size (to ->new_chunk_sectors)
4426 	 * - raid_disks (by delta_disks)
4427 	 * or when trying to restart a reshape that was ongoing.
4428 	 *
4429 	 * We need to validate the request and possibly allocate
4430 	 * space if that might be an issue later.
4431 	 *
4432 	 * Currently we reject any reshape of a 'far' mode array,
4433 	 * allow chunk size to change if new is generally acceptable,
4434 	 * allow raid_disks to increase, and allow
4435 	 * a switch between 'near' mode and 'offset' mode.
4436 	 */
4437 	struct r10conf *conf = mddev->private;
4438 	struct geom geo;
4439 
4440 	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
4441 		return -EINVAL;
4442 
4443 	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
4444 		/* mustn't change number of copies */
4445 		return -EINVAL;
4446 	if (geo.far_copies > 1 && !geo.far_offset)
4447 		/* Cannot switch to 'far' mode */
4448 		return -EINVAL;
4449 
4450 	if (mddev->array_sectors & geo.chunk_mask)
4451 			/* not factor of array size */
4452 			return -EINVAL;
4453 
4454 	if (!enough(conf, -1))
4455 		return -EINVAL;
4456 
4457 	kfree(conf->mirrors_new);
4458 	conf->mirrors_new = NULL;
4459 	if (mddev->delta_disks > 0) {
4460 		/* allocate new 'mirrors' list */
4461 		conf->mirrors_new =
4462 			kcalloc(mddev->raid_disks + mddev->delta_disks,
4463 				sizeof(struct raid10_info),
4464 				GFP_KERNEL);
4465 		if (!conf->mirrors_new)
4466 			return -ENOMEM;
4467 	}
4468 	return 0;
4469 }
4470 
4471 /*
4472  * Need to check if array has failed when deciding whether to:
4473  *  - start an array
4474  *  - remove non-faulty devices
4475  *  - add a spare
4476  *  - allow a reshape
4477  * This determination is simple when no reshape is happening.
4478  * However if there is a reshape, we need to carefully check
4479  * both the before and after sections.
4480  * This is because some failed devices may only affect one
4481  * of the two sections, and some non-in_sync devices may
4482  * be insync in the section most affected by failed devices.
4483  */
calc_degraded(struct r10conf * conf)4484 static int calc_degraded(struct r10conf *conf)
4485 {
4486 	int degraded, degraded2;
4487 	int i;
4488 
4489 	rcu_read_lock();
4490 	degraded = 0;
4491 	/* 'prev' section first */
4492 	for (i = 0; i < conf->prev.raid_disks; i++) {
4493 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
4494 		if (!rdev || test_bit(Faulty, &rdev->flags))
4495 			degraded++;
4496 		else if (!test_bit(In_sync, &rdev->flags))
4497 			/* When we can reduce the number of devices in
4498 			 * an array, this might not contribute to
4499 			 * 'degraded'.  It does now.
4500 			 */
4501 			degraded++;
4502 	}
4503 	rcu_read_unlock();
4504 	if (conf->geo.raid_disks == conf->prev.raid_disks)
4505 		return degraded;
4506 	rcu_read_lock();
4507 	degraded2 = 0;
4508 	for (i = 0; i < conf->geo.raid_disks; i++) {
4509 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
4510 		if (!rdev || test_bit(Faulty, &rdev->flags))
4511 			degraded2++;
4512 		else if (!test_bit(In_sync, &rdev->flags)) {
4513 			/* If reshape is increasing the number of devices,
4514 			 * this section has already been recovered, so
4515 			 * it doesn't contribute to degraded.
4516 			 * else it does.
4517 			 */
4518 			if (conf->geo.raid_disks <= conf->prev.raid_disks)
4519 				degraded2++;
4520 		}
4521 	}
4522 	rcu_read_unlock();
4523 	if (degraded2 > degraded)
4524 		return degraded2;
4525 	return degraded;
4526 }
4527 
raid10_start_reshape(struct mddev * mddev)4528 static int raid10_start_reshape(struct mddev *mddev)
4529 {
4530 	/* A 'reshape' has been requested. This commits
4531 	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
4532 	 * This also checks if there are enough spares and adds them
4533 	 * to the array.
4534 	 * We currently require enough spares to make the final
4535 	 * array non-degraded.  We also require that the difference
4536 	 * between old and new data_offset - on each device - is
4537 	 * enough that we never risk over-writing.
4538 	 */
4539 
4540 	unsigned long before_length, after_length;
4541 	sector_t min_offset_diff = 0;
4542 	int first = 1;
4543 	struct geom new;
4544 	struct r10conf *conf = mddev->private;
4545 	struct md_rdev *rdev;
4546 	int spares = 0;
4547 	int ret;
4548 
4549 	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
4550 		return -EBUSY;
4551 
4552 	if (setup_geo(&new, mddev, geo_start) != conf->copies)
4553 		return -EINVAL;
4554 
4555 	before_length = ((1 << conf->prev.chunk_shift) *
4556 			 conf->prev.far_copies);
4557 	after_length = ((1 << conf->geo.chunk_shift) *
4558 			conf->geo.far_copies);
4559 
4560 	rdev_for_each(rdev, mddev) {
4561 		if (!test_bit(In_sync, &rdev->flags)
4562 		    && !test_bit(Faulty, &rdev->flags))
4563 			spares++;
4564 		if (rdev->raid_disk >= 0) {
4565 			long long diff = (rdev->new_data_offset
4566 					  - rdev->data_offset);
4567 			if (!mddev->reshape_backwards)
4568 				diff = -diff;
4569 			if (diff < 0)
4570 				diff = 0;
4571 			if (first || diff < min_offset_diff)
4572 				min_offset_diff = diff;
4573 			first = 0;
4574 		}
4575 	}
4576 
4577 	if (max(before_length, after_length) > min_offset_diff)
4578 		return -EINVAL;
4579 
4580 	if (spares < mddev->delta_disks)
4581 		return -EINVAL;
4582 
4583 	conf->offset_diff = min_offset_diff;
4584 	spin_lock_irq(&conf->device_lock);
4585 	if (conf->mirrors_new) {
4586 		memcpy(conf->mirrors_new, conf->mirrors,
4587 		       sizeof(struct raid10_info)*conf->prev.raid_disks);
4588 		smp_mb();
4589 		kfree(conf->mirrors_old);
4590 		conf->mirrors_old = conf->mirrors;
4591 		conf->mirrors = conf->mirrors_new;
4592 		conf->mirrors_new = NULL;
4593 	}
4594 	setup_geo(&conf->geo, mddev, geo_start);
4595 	smp_mb();
4596 	if (mddev->reshape_backwards) {
4597 		sector_t size = raid10_size(mddev, 0, 0);
4598 		if (size < mddev->array_sectors) {
4599 			spin_unlock_irq(&conf->device_lock);
4600 			pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
4601 				mdname(mddev));
4602 			return -EINVAL;
4603 		}
4604 		mddev->resync_max_sectors = size;
4605 		conf->reshape_progress = size;
4606 	} else
4607 		conf->reshape_progress = 0;
4608 	conf->reshape_safe = conf->reshape_progress;
4609 	spin_unlock_irq(&conf->device_lock);
4610 
4611 	if (mddev->delta_disks && mddev->bitmap) {
4612 		struct mdp_superblock_1 *sb = NULL;
4613 		sector_t oldsize, newsize;
4614 
4615 		oldsize = raid10_size(mddev, 0, 0);
4616 		newsize = raid10_size(mddev, 0, conf->geo.raid_disks);
4617 
4618 		if (!mddev_is_clustered(mddev)) {
4619 			ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0);
4620 			if (ret)
4621 				goto abort;
4622 			else
4623 				goto out;
4624 		}
4625 
4626 		rdev_for_each(rdev, mddev) {
4627 			if (rdev->raid_disk > -1 &&
4628 			    !test_bit(Faulty, &rdev->flags))
4629 				sb = page_address(rdev->sb_page);
4630 		}
4631 
4632 		/*
4633 		 * some node is already performing reshape, and no need to
4634 		 * call md_bitmap_resize again since it should be called when
4635 		 * receiving BITMAP_RESIZE msg
4636 		 */
4637 		if ((sb && (le32_to_cpu(sb->feature_map) &
4638 			    MD_FEATURE_RESHAPE_ACTIVE)) || (oldsize == newsize))
4639 			goto out;
4640 
4641 		ret = md_bitmap_resize(mddev->bitmap, newsize, 0, 0);
4642 		if (ret)
4643 			goto abort;
4644 
4645 		ret = md_cluster_ops->resize_bitmaps(mddev, newsize, oldsize);
4646 		if (ret) {
4647 			md_bitmap_resize(mddev->bitmap, oldsize, 0, 0);
4648 			goto abort;
4649 		}
4650 	}
4651 out:
4652 	if (mddev->delta_disks > 0) {
4653 		rdev_for_each(rdev, mddev)
4654 			if (rdev->raid_disk < 0 &&
4655 			    !test_bit(Faulty, &rdev->flags)) {
4656 				if (raid10_add_disk(mddev, rdev) == 0) {
4657 					if (rdev->raid_disk >=
4658 					    conf->prev.raid_disks)
4659 						set_bit(In_sync, &rdev->flags);
4660 					else
4661 						rdev->recovery_offset = 0;
4662 
4663 					/* Failure here is OK */
4664 					sysfs_link_rdev(mddev, rdev);
4665 				}
4666 			} else if (rdev->raid_disk >= conf->prev.raid_disks
4667 				   && !test_bit(Faulty, &rdev->flags)) {
4668 				/* This is a spare that was manually added */
4669 				set_bit(In_sync, &rdev->flags);
4670 			}
4671 	}
4672 	/* When a reshape changes the number of devices,
4673 	 * ->degraded is measured against the larger of the
4674 	 * pre and  post numbers.
4675 	 */
4676 	spin_lock_irq(&conf->device_lock);
4677 	mddev->degraded = calc_degraded(conf);
4678 	spin_unlock_irq(&conf->device_lock);
4679 	mddev->raid_disks = conf->geo.raid_disks;
4680 	mddev->reshape_position = conf->reshape_progress;
4681 	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
4682 
4683 	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4684 	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4685 	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
4686 	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4687 	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4688 
4689 	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4690 						"reshape");
4691 	if (!mddev->sync_thread) {
4692 		ret = -EAGAIN;
4693 		goto abort;
4694 	}
4695 	conf->reshape_checkpoint = jiffies;
4696 	md_wakeup_thread(mddev->sync_thread);
4697 	md_new_event();
4698 	return 0;
4699 
4700 abort:
4701 	mddev->recovery = 0;
4702 	spin_lock_irq(&conf->device_lock);
4703 	conf->geo = conf->prev;
4704 	mddev->raid_disks = conf->geo.raid_disks;
4705 	rdev_for_each(rdev, mddev)
4706 		rdev->new_data_offset = rdev->data_offset;
4707 	smp_wmb();
4708 	conf->reshape_progress = MaxSector;
4709 	conf->reshape_safe = MaxSector;
4710 	mddev->reshape_position = MaxSector;
4711 	spin_unlock_irq(&conf->device_lock);
4712 	return ret;
4713 }
4714 
4715 /* Calculate the last device-address that could contain
4716  * any block from the chunk that includes the array-address 's'
4717  * and report the next address.
4718  * i.e. the address returned will be chunk-aligned and after
4719  * any data that is in the chunk containing 's'.
4720  */
last_dev_address(sector_t s,struct geom * geo)4721 static sector_t last_dev_address(sector_t s, struct geom *geo)
4722 {
4723 	s = (s | geo->chunk_mask) + 1;
4724 	s >>= geo->chunk_shift;
4725 	s *= geo->near_copies;
4726 	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4727 	s *= geo->far_copies;
4728 	s <<= geo->chunk_shift;
4729 	return s;
4730 }
4731 
4732 /* Calculate the first device-address that could contain
4733  * any block from the chunk that includes the array-address 's'.
4734  * This too will be the start of a chunk
4735  */
first_dev_address(sector_t s,struct geom * geo)4736 static sector_t first_dev_address(sector_t s, struct geom *geo)
4737 {
4738 	s >>= geo->chunk_shift;
4739 	s *= geo->near_copies;
4740 	sector_div(s, geo->raid_disks);
4741 	s *= geo->far_copies;
4742 	s <<= geo->chunk_shift;
4743 	return s;
4744 }
4745 
reshape_request(struct mddev * mddev,sector_t sector_nr,int * skipped)4746 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4747 				int *skipped)
4748 {
4749 	/* We simply copy at most one chunk (smallest of old and new)
4750 	 * at a time, possibly less if that exceeds RESYNC_PAGES,
4751 	 * or we hit a bad block or something.
4752 	 * This might mean we pause for normal IO in the middle of
4753 	 * a chunk, but that is not a problem as mddev->reshape_position
4754 	 * can record any location.
4755 	 *
4756 	 * If we will want to write to a location that isn't
4757 	 * yet recorded as 'safe' (i.e. in metadata on disk) then
4758 	 * we need to flush all reshape requests and update the metadata.
4759 	 *
4760 	 * When reshaping forwards (e.g. to more devices), we interpret
4761 	 * 'safe' as the earliest block which might not have been copied
4762 	 * down yet.  We divide this by previous stripe size and multiply
4763 	 * by previous stripe length to get lowest device offset that we
4764 	 * cannot write to yet.
4765 	 * We interpret 'sector_nr' as an address that we want to write to.
4766 	 * From this we use last_device_address() to find where we might
4767 	 * write to, and first_device_address on the  'safe' position.
4768 	 * If this 'next' write position is after the 'safe' position,
4769 	 * we must update the metadata to increase the 'safe' position.
4770 	 *
4771 	 * When reshaping backwards, we round in the opposite direction
4772 	 * and perform the reverse test:  next write position must not be
4773 	 * less than current safe position.
4774 	 *
4775 	 * In all this the minimum difference in data offsets
4776 	 * (conf->offset_diff - always positive) allows a bit of slack,
4777 	 * so next can be after 'safe', but not by more than offset_diff
4778 	 *
4779 	 * We need to prepare all the bios here before we start any IO
4780 	 * to ensure the size we choose is acceptable to all devices.
4781 	 * The means one for each copy for write-out and an extra one for
4782 	 * read-in.
4783 	 * We store the read-in bio in ->master_bio and the others in
4784 	 * ->devs[x].bio and ->devs[x].repl_bio.
4785 	 */
4786 	struct r10conf *conf = mddev->private;
4787 	struct r10bio *r10_bio;
4788 	sector_t next, safe, last;
4789 	int max_sectors;
4790 	int nr_sectors;
4791 	int s;
4792 	struct md_rdev *rdev;
4793 	int need_flush = 0;
4794 	struct bio *blist;
4795 	struct bio *bio, *read_bio;
4796 	int sectors_done = 0;
4797 	struct page **pages;
4798 
4799 	if (sector_nr == 0) {
4800 		/* If restarting in the middle, skip the initial sectors */
4801 		if (mddev->reshape_backwards &&
4802 		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
4803 			sector_nr = (raid10_size(mddev, 0, 0)
4804 				     - conf->reshape_progress);
4805 		} else if (!mddev->reshape_backwards &&
4806 			   conf->reshape_progress > 0)
4807 			sector_nr = conf->reshape_progress;
4808 		if (sector_nr) {
4809 			mddev->curr_resync_completed = sector_nr;
4810 			sysfs_notify_dirent_safe(mddev->sysfs_completed);
4811 			*skipped = 1;
4812 			return sector_nr;
4813 		}
4814 	}
4815 
4816 	/* We don't use sector_nr to track where we are up to
4817 	 * as that doesn't work well for ->reshape_backwards.
4818 	 * So just use ->reshape_progress.
4819 	 */
4820 	if (mddev->reshape_backwards) {
4821 		/* 'next' is the earliest device address that we might
4822 		 * write to for this chunk in the new layout
4823 		 */
4824 		next = first_dev_address(conf->reshape_progress - 1,
4825 					 &conf->geo);
4826 
4827 		/* 'safe' is the last device address that we might read from
4828 		 * in the old layout after a restart
4829 		 */
4830 		safe = last_dev_address(conf->reshape_safe - 1,
4831 					&conf->prev);
4832 
4833 		if (next + conf->offset_diff < safe)
4834 			need_flush = 1;
4835 
4836 		last = conf->reshape_progress - 1;
4837 		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4838 					       & conf->prev.chunk_mask);
4839 		if (sector_nr + RESYNC_SECTORS < last)
4840 			sector_nr = last + 1 - RESYNC_SECTORS;
4841 	} else {
4842 		/* 'next' is after the last device address that we
4843 		 * might write to for this chunk in the new layout
4844 		 */
4845 		next = last_dev_address(conf->reshape_progress, &conf->geo);
4846 
4847 		/* 'safe' is the earliest device address that we might
4848 		 * read from in the old layout after a restart
4849 		 */
4850 		safe = first_dev_address(conf->reshape_safe, &conf->prev);
4851 
4852 		/* Need to update metadata if 'next' might be beyond 'safe'
4853 		 * as that would possibly corrupt data
4854 		 */
4855 		if (next > safe + conf->offset_diff)
4856 			need_flush = 1;
4857 
4858 		sector_nr = conf->reshape_progress;
4859 		last  = sector_nr | (conf->geo.chunk_mask
4860 				     & conf->prev.chunk_mask);
4861 
4862 		if (sector_nr + RESYNC_SECTORS <= last)
4863 			last = sector_nr + RESYNC_SECTORS - 1;
4864 	}
4865 
4866 	if (need_flush ||
4867 	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4868 		/* Need to update reshape_position in metadata */
4869 		wait_barrier(conf, false);
4870 		mddev->reshape_position = conf->reshape_progress;
4871 		if (mddev->reshape_backwards)
4872 			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
4873 				- conf->reshape_progress;
4874 		else
4875 			mddev->curr_resync_completed = conf->reshape_progress;
4876 		conf->reshape_checkpoint = jiffies;
4877 		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
4878 		md_wakeup_thread(mddev->thread);
4879 		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4880 			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4881 		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
4882 			allow_barrier(conf);
4883 			return sectors_done;
4884 		}
4885 		conf->reshape_safe = mddev->reshape_position;
4886 		allow_barrier(conf);
4887 	}
4888 
4889 	raise_barrier(conf, 0);
4890 read_more:
4891 	/* Now schedule reads for blocks from sector_nr to last */
4892 	r10_bio = raid10_alloc_init_r10buf(conf);
4893 	r10_bio->state = 0;
4894 	raise_barrier(conf, 1);
4895 	atomic_set(&r10_bio->remaining, 0);
4896 	r10_bio->mddev = mddev;
4897 	r10_bio->sector = sector_nr;
4898 	set_bit(R10BIO_IsReshape, &r10_bio->state);
4899 	r10_bio->sectors = last - sector_nr + 1;
4900 	rdev = read_balance(conf, r10_bio, &max_sectors);
4901 	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4902 
4903 	if (!rdev) {
4904 		/* Cannot read from here, so need to record bad blocks
4905 		 * on all the target devices.
4906 		 */
4907 		// FIXME
4908 		mempool_free(r10_bio, &conf->r10buf_pool);
4909 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
4910 		return sectors_done;
4911 	}
4912 
4913 	read_bio = bio_alloc_bioset(rdev->bdev, RESYNC_PAGES, REQ_OP_READ,
4914 				    GFP_KERNEL, &mddev->bio_set);
4915 	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4916 			       + rdev->data_offset);
4917 	read_bio->bi_private = r10_bio;
4918 	read_bio->bi_end_io = end_reshape_read;
4919 	r10_bio->master_bio = read_bio;
4920 	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4921 
4922 	/*
4923 	 * Broadcast RESYNC message to other nodes, so all nodes would not
4924 	 * write to the region to avoid conflict.
4925 	*/
4926 	if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) {
4927 		struct mdp_superblock_1 *sb = NULL;
4928 		int sb_reshape_pos = 0;
4929 
4930 		conf->cluster_sync_low = sector_nr;
4931 		conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS;
4932 		sb = page_address(rdev->sb_page);
4933 		if (sb) {
4934 			sb_reshape_pos = le64_to_cpu(sb->reshape_position);
4935 			/*
4936 			 * Set cluster_sync_low again if next address for array
4937 			 * reshape is less than cluster_sync_low. Since we can't
4938 			 * update cluster_sync_low until it has finished reshape.
4939 			 */
4940 			if (sb_reshape_pos < conf->cluster_sync_low)
4941 				conf->cluster_sync_low = sb_reshape_pos;
4942 		}
4943 
4944 		md_cluster_ops->resync_info_update(mddev, conf->cluster_sync_low,
4945 							  conf->cluster_sync_high);
4946 	}
4947 
4948 	/* Now find the locations in the new layout */
4949 	__raid10_find_phys(&conf->geo, r10_bio);
4950 
4951 	blist = read_bio;
4952 	read_bio->bi_next = NULL;
4953 
4954 	rcu_read_lock();
4955 	for (s = 0; s < conf->copies*2; s++) {
4956 		struct bio *b;
4957 		int d = r10_bio->devs[s/2].devnum;
4958 		struct md_rdev *rdev2;
4959 		if (s&1) {
4960 			rdev2 = rcu_dereference(conf->mirrors[d].replacement);
4961 			b = r10_bio->devs[s/2].repl_bio;
4962 		} else {
4963 			rdev2 = rcu_dereference(conf->mirrors[d].rdev);
4964 			b = r10_bio->devs[s/2].bio;
4965 		}
4966 		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4967 			continue;
4968 
4969 		bio_set_dev(b, rdev2->bdev);
4970 		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
4971 			rdev2->new_data_offset;
4972 		b->bi_end_io = end_reshape_write;
4973 		bio_set_op_attrs(b, REQ_OP_WRITE, 0);
4974 		b->bi_next = blist;
4975 		blist = b;
4976 	}
4977 
4978 	/* Now add as many pages as possible to all of these bios. */
4979 
4980 	nr_sectors = 0;
4981 	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
4982 	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4983 		struct page *page = pages[s / (PAGE_SIZE >> 9)];
4984 		int len = (max_sectors - s) << 9;
4985 		if (len > PAGE_SIZE)
4986 			len = PAGE_SIZE;
4987 		for (bio = blist; bio ; bio = bio->bi_next) {
4988 			/*
4989 			 * won't fail because the vec table is big enough
4990 			 * to hold all these pages
4991 			 */
4992 			bio_add_page(bio, page, len, 0);
4993 		}
4994 		sector_nr += len >> 9;
4995 		nr_sectors += len >> 9;
4996 	}
4997 	rcu_read_unlock();
4998 	r10_bio->sectors = nr_sectors;
4999 
5000 	/* Now submit the read */
5001 	md_sync_acct_bio(read_bio, r10_bio->sectors);
5002 	atomic_inc(&r10_bio->remaining);
5003 	read_bio->bi_next = NULL;
5004 	submit_bio_noacct(read_bio);
5005 	sectors_done += nr_sectors;
5006 	if (sector_nr <= last)
5007 		goto read_more;
5008 
5009 	lower_barrier(conf);
5010 
5011 	/* Now that we have done the whole section we can
5012 	 * update reshape_progress
5013 	 */
5014 	if (mddev->reshape_backwards)
5015 		conf->reshape_progress -= sectors_done;
5016 	else
5017 		conf->reshape_progress += sectors_done;
5018 
5019 	return sectors_done;
5020 }
5021 
5022 static void end_reshape_request(struct r10bio *r10_bio);
5023 static int handle_reshape_read_error(struct mddev *mddev,
5024 				     struct r10bio *r10_bio);
reshape_request_write(struct mddev * mddev,struct r10bio * r10_bio)5025 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
5026 {
5027 	/* Reshape read completed.  Hopefully we have a block
5028 	 * to write out.
5029 	 * If we got a read error then we do sync 1-page reads from
5030 	 * elsewhere until we find the data - or give up.
5031 	 */
5032 	struct r10conf *conf = mddev->private;
5033 	int s;
5034 
5035 	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
5036 		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
5037 			/* Reshape has been aborted */
5038 			md_done_sync(mddev, r10_bio->sectors, 0);
5039 			return;
5040 		}
5041 
5042 	/* We definitely have the data in the pages, schedule the
5043 	 * writes.
5044 	 */
5045 	atomic_set(&r10_bio->remaining, 1);
5046 	for (s = 0; s < conf->copies*2; s++) {
5047 		struct bio *b;
5048 		int d = r10_bio->devs[s/2].devnum;
5049 		struct md_rdev *rdev;
5050 		rcu_read_lock();
5051 		if (s&1) {
5052 			rdev = rcu_dereference(conf->mirrors[d].replacement);
5053 			b = r10_bio->devs[s/2].repl_bio;
5054 		} else {
5055 			rdev = rcu_dereference(conf->mirrors[d].rdev);
5056 			b = r10_bio->devs[s/2].bio;
5057 		}
5058 		if (!rdev || test_bit(Faulty, &rdev->flags)) {
5059 			rcu_read_unlock();
5060 			continue;
5061 		}
5062 		atomic_inc(&rdev->nr_pending);
5063 		rcu_read_unlock();
5064 		md_sync_acct_bio(b, r10_bio->sectors);
5065 		atomic_inc(&r10_bio->remaining);
5066 		b->bi_next = NULL;
5067 		submit_bio_noacct(b);
5068 	}
5069 	end_reshape_request(r10_bio);
5070 }
5071 
end_reshape(struct r10conf * conf)5072 static void end_reshape(struct r10conf *conf)
5073 {
5074 	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
5075 		return;
5076 
5077 	spin_lock_irq(&conf->device_lock);
5078 	conf->prev = conf->geo;
5079 	md_finish_reshape(conf->mddev);
5080 	smp_wmb();
5081 	conf->reshape_progress = MaxSector;
5082 	conf->reshape_safe = MaxSector;
5083 	spin_unlock_irq(&conf->device_lock);
5084 
5085 	if (conf->mddev->queue)
5086 		raid10_set_io_opt(conf);
5087 	conf->fullsync = 0;
5088 }
5089 
raid10_update_reshape_pos(struct mddev * mddev)5090 static void raid10_update_reshape_pos(struct mddev *mddev)
5091 {
5092 	struct r10conf *conf = mddev->private;
5093 	sector_t lo, hi;
5094 
5095 	md_cluster_ops->resync_info_get(mddev, &lo, &hi);
5096 	if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo))
5097 	    || mddev->reshape_position == MaxSector)
5098 		conf->reshape_progress = mddev->reshape_position;
5099 	else
5100 		WARN_ON_ONCE(1);
5101 }
5102 
handle_reshape_read_error(struct mddev * mddev,struct r10bio * r10_bio)5103 static int handle_reshape_read_error(struct mddev *mddev,
5104 				     struct r10bio *r10_bio)
5105 {
5106 	/* Use sync reads to get the blocks from somewhere else */
5107 	int sectors = r10_bio->sectors;
5108 	struct r10conf *conf = mddev->private;
5109 	struct r10bio *r10b;
5110 	int slot = 0;
5111 	int idx = 0;
5112 	struct page **pages;
5113 
5114 	r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO);
5115 	if (!r10b) {
5116 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
5117 		return -ENOMEM;
5118 	}
5119 
5120 	/* reshape IOs share pages from .devs[0].bio */
5121 	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
5122 
5123 	r10b->sector = r10_bio->sector;
5124 	__raid10_find_phys(&conf->prev, r10b);
5125 
5126 	while (sectors) {
5127 		int s = sectors;
5128 		int success = 0;
5129 		int first_slot = slot;
5130 
5131 		if (s > (PAGE_SIZE >> 9))
5132 			s = PAGE_SIZE >> 9;
5133 
5134 		rcu_read_lock();
5135 		while (!success) {
5136 			int d = r10b->devs[slot].devnum;
5137 			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
5138 			sector_t addr;
5139 			if (rdev == NULL ||
5140 			    test_bit(Faulty, &rdev->flags) ||
5141 			    !test_bit(In_sync, &rdev->flags))
5142 				goto failed;
5143 
5144 			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
5145 			atomic_inc(&rdev->nr_pending);
5146 			rcu_read_unlock();
5147 			success = sync_page_io(rdev,
5148 					       addr,
5149 					       s << 9,
5150 					       pages[idx],
5151 					       REQ_OP_READ, false);
5152 			rdev_dec_pending(rdev, mddev);
5153 			rcu_read_lock();
5154 			if (success)
5155 				break;
5156 		failed:
5157 			slot++;
5158 			if (slot >= conf->copies)
5159 				slot = 0;
5160 			if (slot == first_slot)
5161 				break;
5162 		}
5163 		rcu_read_unlock();
5164 		if (!success) {
5165 			/* couldn't read this block, must give up */
5166 			set_bit(MD_RECOVERY_INTR,
5167 				&mddev->recovery);
5168 			kfree(r10b);
5169 			return -EIO;
5170 		}
5171 		sectors -= s;
5172 		idx++;
5173 	}
5174 	kfree(r10b);
5175 	return 0;
5176 }
5177 
end_reshape_write(struct bio * bio)5178 static void end_reshape_write(struct bio *bio)
5179 {
5180 	struct r10bio *r10_bio = get_resync_r10bio(bio);
5181 	struct mddev *mddev = r10_bio->mddev;
5182 	struct r10conf *conf = mddev->private;
5183 	int d;
5184 	int slot;
5185 	int repl;
5186 	struct md_rdev *rdev = NULL;
5187 
5188 	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
5189 	if (repl)
5190 		rdev = conf->mirrors[d].replacement;
5191 	if (!rdev) {
5192 		smp_mb();
5193 		rdev = conf->mirrors[d].rdev;
5194 	}
5195 
5196 	if (bio->bi_status) {
5197 		/* FIXME should record badblock */
5198 		md_error(mddev, rdev);
5199 	}
5200 
5201 	rdev_dec_pending(rdev, mddev);
5202 	end_reshape_request(r10_bio);
5203 }
5204 
end_reshape_request(struct r10bio * r10_bio)5205 static void end_reshape_request(struct r10bio *r10_bio)
5206 {
5207 	if (!atomic_dec_and_test(&r10_bio->remaining))
5208 		return;
5209 	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
5210 	bio_put(r10_bio->master_bio);
5211 	put_buf(r10_bio);
5212 }
5213 
raid10_finish_reshape(struct mddev * mddev)5214 static void raid10_finish_reshape(struct mddev *mddev)
5215 {
5216 	struct r10conf *conf = mddev->private;
5217 
5218 	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5219 		return;
5220 
5221 	if (mddev->delta_disks > 0) {
5222 		if (mddev->recovery_cp > mddev->resync_max_sectors) {
5223 			mddev->recovery_cp = mddev->resync_max_sectors;
5224 			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5225 		}
5226 		mddev->resync_max_sectors = mddev->array_sectors;
5227 	} else {
5228 		int d;
5229 		rcu_read_lock();
5230 		for (d = conf->geo.raid_disks ;
5231 		     d < conf->geo.raid_disks - mddev->delta_disks;
5232 		     d++) {
5233 			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
5234 			if (rdev)
5235 				clear_bit(In_sync, &rdev->flags);
5236 			rdev = rcu_dereference(conf->mirrors[d].replacement);
5237 			if (rdev)
5238 				clear_bit(In_sync, &rdev->flags);
5239 		}
5240 		rcu_read_unlock();
5241 	}
5242 	mddev->layout = mddev->new_layout;
5243 	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
5244 	mddev->reshape_position = MaxSector;
5245 	mddev->delta_disks = 0;
5246 	mddev->reshape_backwards = 0;
5247 }
5248 
5249 static struct md_personality raid10_personality =
5250 {
5251 	.name		= "raid10",
5252 	.level		= 10,
5253 	.owner		= THIS_MODULE,
5254 	.make_request	= raid10_make_request,
5255 	.run		= raid10_run,
5256 	.free		= raid10_free,
5257 	.status		= raid10_status,
5258 	.error_handler	= raid10_error,
5259 	.hot_add_disk	= raid10_add_disk,
5260 	.hot_remove_disk= raid10_remove_disk,
5261 	.spare_active	= raid10_spare_active,
5262 	.sync_request	= raid10_sync_request,
5263 	.quiesce	= raid10_quiesce,
5264 	.size		= raid10_size,
5265 	.resize		= raid10_resize,
5266 	.takeover	= raid10_takeover,
5267 	.check_reshape	= raid10_check_reshape,
5268 	.start_reshape	= raid10_start_reshape,
5269 	.finish_reshape	= raid10_finish_reshape,
5270 	.update_reshape_pos = raid10_update_reshape_pos,
5271 };
5272 
raid_init(void)5273 static int __init raid_init(void)
5274 {
5275 	return register_md_personality(&raid10_personality);
5276 }
5277 
raid_exit(void)5278 static void raid_exit(void)
5279 {
5280 	unregister_md_personality(&raid10_personality);
5281 }
5282 
5283 module_init(raid_init);
5284 module_exit(raid_exit);
5285 MODULE_LICENSE("GPL");
5286 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
5287 MODULE_ALIAS("md-personality-9"); /* RAID10 */
5288 MODULE_ALIAS("md-raid10");
5289 MODULE_ALIAS("md-level-10");
5290