1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Block multiqueue core code
4  *
5  * Copyright (C) 2013-2014 Jens Axboe
6  * Copyright (C) 2013-2014 Christoph Hellwig
7  */
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/backing-dev.h>
11 #include <linux/bio.h>
12 #include <linux/blkdev.h>
13 #include <linux/blk-integrity.h>
14 #include <linux/kmemleak.h>
15 #include <linux/mm.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/workqueue.h>
19 #include <linux/smp.h>
20 #include <linux/interrupt.h>
21 #include <linux/llist.h>
22 #include <linux/cpu.h>
23 #include <linux/cache.h>
24 #include <linux/sched/sysctl.h>
25 #include <linux/sched/topology.h>
26 #include <linux/sched/signal.h>
27 #include <linux/delay.h>
28 #include <linux/crash_dump.h>
29 #include <linux/prefetch.h>
30 #include <linux/blk-crypto.h>
31 #include <linux/part_stat.h>
32 
33 #include <trace/events/block.h>
34 
35 #include <linux/t10-pi.h>
36 #include "blk.h"
37 #include "blk-mq.h"
38 #include "blk-mq-debugfs.h"
39 #include "blk-pm.h"
40 #include "blk-stat.h"
41 #include "blk-mq-sched.h"
42 #include "blk-rq-qos.h"
43 #include "blk-ioprio.h"
44 
45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
47 
48 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
49 static void blk_mq_request_bypass_insert(struct request *rq,
50 		blk_insert_t flags);
51 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
52 		struct list_head *list);
53 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
54 			 struct io_comp_batch *iob, unsigned int flags);
55 
56 /*
57  * Check if any of the ctx, dispatch list or elevator
58  * have pending work in this hardware queue.
59  */
blk_mq_hctx_has_pending(struct blk_mq_hw_ctx * hctx)60 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
61 {
62 	return !list_empty_careful(&hctx->dispatch) ||
63 		sbitmap_any_bit_set(&hctx->ctx_map) ||
64 			blk_mq_sched_has_work(hctx);
65 }
66 
67 /*
68  * Mark this ctx as having pending work in this hardware queue
69  */
blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx)70 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
71 				     struct blk_mq_ctx *ctx)
72 {
73 	const int bit = ctx->index_hw[hctx->type];
74 
75 	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
76 		sbitmap_set_bit(&hctx->ctx_map, bit);
77 }
78 
blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx)79 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
80 				      struct blk_mq_ctx *ctx)
81 {
82 	const int bit = ctx->index_hw[hctx->type];
83 
84 	sbitmap_clear_bit(&hctx->ctx_map, bit);
85 }
86 
87 struct mq_inflight {
88 	struct block_device *part;
89 	unsigned int inflight[2];
90 };
91 
blk_mq_check_inflight(struct request * rq,void * priv)92 static bool blk_mq_check_inflight(struct request *rq, void *priv)
93 {
94 	struct mq_inflight *mi = priv;
95 
96 	if (rq->part && blk_do_io_stat(rq) &&
97 	    (!mi->part->bd_partno || rq->part == mi->part) &&
98 	    blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
99 		mi->inflight[rq_data_dir(rq)]++;
100 
101 	return true;
102 }
103 
blk_mq_in_flight(struct request_queue * q,struct block_device * part)104 unsigned int blk_mq_in_flight(struct request_queue *q,
105 		struct block_device *part)
106 {
107 	struct mq_inflight mi = { .part = part };
108 
109 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
110 
111 	return mi.inflight[0] + mi.inflight[1];
112 }
113 
blk_mq_in_flight_rw(struct request_queue * q,struct block_device * part,unsigned int inflight[2])114 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
115 		unsigned int inflight[2])
116 {
117 	struct mq_inflight mi = { .part = part };
118 
119 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
120 	inflight[0] = mi.inflight[0];
121 	inflight[1] = mi.inflight[1];
122 }
123 
blk_freeze_queue_start(struct request_queue * q)124 void blk_freeze_queue_start(struct request_queue *q)
125 {
126 	mutex_lock(&q->mq_freeze_lock);
127 	if (++q->mq_freeze_depth == 1) {
128 		percpu_ref_kill(&q->q_usage_counter);
129 		mutex_unlock(&q->mq_freeze_lock);
130 		if (queue_is_mq(q))
131 			blk_mq_run_hw_queues(q, false);
132 	} else {
133 		mutex_unlock(&q->mq_freeze_lock);
134 	}
135 }
136 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
137 
blk_mq_freeze_queue_wait(struct request_queue * q)138 void blk_mq_freeze_queue_wait(struct request_queue *q)
139 {
140 	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
141 }
142 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
143 
blk_mq_freeze_queue_wait_timeout(struct request_queue * q,unsigned long timeout)144 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
145 				     unsigned long timeout)
146 {
147 	return wait_event_timeout(q->mq_freeze_wq,
148 					percpu_ref_is_zero(&q->q_usage_counter),
149 					timeout);
150 }
151 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
152 
153 /*
154  * Guarantee no request is in use, so we can change any data structure of
155  * the queue afterward.
156  */
blk_freeze_queue(struct request_queue * q)157 void blk_freeze_queue(struct request_queue *q)
158 {
159 	/*
160 	 * In the !blk_mq case we are only calling this to kill the
161 	 * q_usage_counter, otherwise this increases the freeze depth
162 	 * and waits for it to return to zero.  For this reason there is
163 	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
164 	 * exported to drivers as the only user for unfreeze is blk_mq.
165 	 */
166 	blk_freeze_queue_start(q);
167 	blk_mq_freeze_queue_wait(q);
168 }
169 
blk_mq_freeze_queue(struct request_queue * q)170 void blk_mq_freeze_queue(struct request_queue *q)
171 {
172 	/*
173 	 * ...just an alias to keep freeze and unfreeze actions balanced
174 	 * in the blk_mq_* namespace
175 	 */
176 	blk_freeze_queue(q);
177 }
178 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
179 
__blk_mq_unfreeze_queue(struct request_queue * q,bool force_atomic)180 void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
181 {
182 	mutex_lock(&q->mq_freeze_lock);
183 	if (force_atomic)
184 		q->q_usage_counter.data->force_atomic = true;
185 	q->mq_freeze_depth--;
186 	WARN_ON_ONCE(q->mq_freeze_depth < 0);
187 	if (!q->mq_freeze_depth) {
188 		percpu_ref_resurrect(&q->q_usage_counter);
189 		wake_up_all(&q->mq_freeze_wq);
190 	}
191 	mutex_unlock(&q->mq_freeze_lock);
192 }
193 
blk_mq_unfreeze_queue(struct request_queue * q)194 void blk_mq_unfreeze_queue(struct request_queue *q)
195 {
196 	__blk_mq_unfreeze_queue(q, false);
197 }
198 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
199 
200 /*
201  * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
202  * mpt3sas driver such that this function can be removed.
203  */
blk_mq_quiesce_queue_nowait(struct request_queue * q)204 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
205 {
206 	unsigned long flags;
207 
208 	spin_lock_irqsave(&q->queue_lock, flags);
209 	if (!q->quiesce_depth++)
210 		blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
211 	spin_unlock_irqrestore(&q->queue_lock, flags);
212 }
213 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
214 
215 /**
216  * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
217  * @set: tag_set to wait on
218  *
219  * Note: it is driver's responsibility for making sure that quiesce has
220  * been started on or more of the request_queues of the tag_set.  This
221  * function only waits for the quiesce on those request_queues that had
222  * the quiesce flag set using blk_mq_quiesce_queue_nowait.
223  */
blk_mq_wait_quiesce_done(struct blk_mq_tag_set * set)224 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
225 {
226 	if (set->flags & BLK_MQ_F_BLOCKING)
227 		synchronize_srcu(set->srcu);
228 	else
229 		synchronize_rcu();
230 }
231 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
232 
233 /**
234  * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
235  * @q: request queue.
236  *
237  * Note: this function does not prevent that the struct request end_io()
238  * callback function is invoked. Once this function is returned, we make
239  * sure no dispatch can happen until the queue is unquiesced via
240  * blk_mq_unquiesce_queue().
241  */
blk_mq_quiesce_queue(struct request_queue * q)242 void blk_mq_quiesce_queue(struct request_queue *q)
243 {
244 	blk_mq_quiesce_queue_nowait(q);
245 	/* nothing to wait for non-mq queues */
246 	if (queue_is_mq(q))
247 		blk_mq_wait_quiesce_done(q->tag_set);
248 }
249 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
250 
251 /*
252  * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
253  * @q: request queue.
254  *
255  * This function recovers queue into the state before quiescing
256  * which is done by blk_mq_quiesce_queue.
257  */
blk_mq_unquiesce_queue(struct request_queue * q)258 void blk_mq_unquiesce_queue(struct request_queue *q)
259 {
260 	unsigned long flags;
261 	bool run_queue = false;
262 
263 	spin_lock_irqsave(&q->queue_lock, flags);
264 	if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
265 		;
266 	} else if (!--q->quiesce_depth) {
267 		blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
268 		run_queue = true;
269 	}
270 	spin_unlock_irqrestore(&q->queue_lock, flags);
271 
272 	/* dispatch requests which are inserted during quiescing */
273 	if (run_queue)
274 		blk_mq_run_hw_queues(q, true);
275 }
276 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
277 
blk_mq_quiesce_tagset(struct blk_mq_tag_set * set)278 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
279 {
280 	struct request_queue *q;
281 
282 	mutex_lock(&set->tag_list_lock);
283 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
284 		if (!blk_queue_skip_tagset_quiesce(q))
285 			blk_mq_quiesce_queue_nowait(q);
286 	}
287 	blk_mq_wait_quiesce_done(set);
288 	mutex_unlock(&set->tag_list_lock);
289 }
290 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
291 
blk_mq_unquiesce_tagset(struct blk_mq_tag_set * set)292 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
293 {
294 	struct request_queue *q;
295 
296 	mutex_lock(&set->tag_list_lock);
297 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
298 		if (!blk_queue_skip_tagset_quiesce(q))
299 			blk_mq_unquiesce_queue(q);
300 	}
301 	mutex_unlock(&set->tag_list_lock);
302 }
303 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
304 
blk_mq_wake_waiters(struct request_queue * q)305 void blk_mq_wake_waiters(struct request_queue *q)
306 {
307 	struct blk_mq_hw_ctx *hctx;
308 	unsigned long i;
309 
310 	queue_for_each_hw_ctx(q, hctx, i)
311 		if (blk_mq_hw_queue_mapped(hctx))
312 			blk_mq_tag_wakeup_all(hctx->tags, true);
313 }
314 
blk_rq_init(struct request_queue * q,struct request * rq)315 void blk_rq_init(struct request_queue *q, struct request *rq)
316 {
317 	memset(rq, 0, sizeof(*rq));
318 
319 	INIT_LIST_HEAD(&rq->queuelist);
320 	rq->q = q;
321 	rq->__sector = (sector_t) -1;
322 	INIT_HLIST_NODE(&rq->hash);
323 	RB_CLEAR_NODE(&rq->rb_node);
324 	rq->tag = BLK_MQ_NO_TAG;
325 	rq->internal_tag = BLK_MQ_NO_TAG;
326 	rq->start_time_ns = ktime_get_ns();
327 	rq->part = NULL;
328 	blk_crypto_rq_set_defaults(rq);
329 }
330 EXPORT_SYMBOL(blk_rq_init);
331 
332 /* Set start and alloc time when the allocated request is actually used */
blk_mq_rq_time_init(struct request * rq,u64 alloc_time_ns)333 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
334 {
335 	if (blk_mq_need_time_stamp(rq))
336 		rq->start_time_ns = ktime_get_ns();
337 	else
338 		rq->start_time_ns = 0;
339 
340 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
341 	if (blk_queue_rq_alloc_time(rq->q))
342 		rq->alloc_time_ns = alloc_time_ns ?: rq->start_time_ns;
343 	else
344 		rq->alloc_time_ns = 0;
345 #endif
346 }
347 
blk_mq_rq_ctx_init(struct blk_mq_alloc_data * data,struct blk_mq_tags * tags,unsigned int tag)348 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
349 		struct blk_mq_tags *tags, unsigned int tag)
350 {
351 	struct blk_mq_ctx *ctx = data->ctx;
352 	struct blk_mq_hw_ctx *hctx = data->hctx;
353 	struct request_queue *q = data->q;
354 	struct request *rq = tags->static_rqs[tag];
355 
356 	rq->q = q;
357 	rq->mq_ctx = ctx;
358 	rq->mq_hctx = hctx;
359 	rq->cmd_flags = data->cmd_flags;
360 
361 	if (data->flags & BLK_MQ_REQ_PM)
362 		data->rq_flags |= RQF_PM;
363 	if (blk_queue_io_stat(q))
364 		data->rq_flags |= RQF_IO_STAT;
365 	rq->rq_flags = data->rq_flags;
366 
367 	if (data->rq_flags & RQF_SCHED_TAGS) {
368 		rq->tag = BLK_MQ_NO_TAG;
369 		rq->internal_tag = tag;
370 	} else {
371 		rq->tag = tag;
372 		rq->internal_tag = BLK_MQ_NO_TAG;
373 	}
374 	rq->timeout = 0;
375 
376 	rq->part = NULL;
377 	rq->io_start_time_ns = 0;
378 	rq->stats_sectors = 0;
379 	rq->nr_phys_segments = 0;
380 #if defined(CONFIG_BLK_DEV_INTEGRITY)
381 	rq->nr_integrity_segments = 0;
382 #endif
383 	rq->end_io = NULL;
384 	rq->end_io_data = NULL;
385 
386 	blk_crypto_rq_set_defaults(rq);
387 	INIT_LIST_HEAD(&rq->queuelist);
388 	/* tag was already set */
389 	WRITE_ONCE(rq->deadline, 0);
390 	req_ref_set(rq, 1);
391 
392 	if (rq->rq_flags & RQF_USE_SCHED) {
393 		struct elevator_queue *e = data->q->elevator;
394 
395 		INIT_HLIST_NODE(&rq->hash);
396 		RB_CLEAR_NODE(&rq->rb_node);
397 
398 		if (e->type->ops.prepare_request)
399 			e->type->ops.prepare_request(rq);
400 	}
401 
402 	return rq;
403 }
404 
405 static inline struct request *
__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data * data)406 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
407 {
408 	unsigned int tag, tag_offset;
409 	struct blk_mq_tags *tags;
410 	struct request *rq;
411 	unsigned long tag_mask;
412 	int i, nr = 0;
413 
414 	tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
415 	if (unlikely(!tag_mask))
416 		return NULL;
417 
418 	tags = blk_mq_tags_from_data(data);
419 	for (i = 0; tag_mask; i++) {
420 		if (!(tag_mask & (1UL << i)))
421 			continue;
422 		tag = tag_offset + i;
423 		prefetch(tags->static_rqs[tag]);
424 		tag_mask &= ~(1UL << i);
425 		rq = blk_mq_rq_ctx_init(data, tags, tag);
426 		rq_list_add(data->cached_rq, rq);
427 		nr++;
428 	}
429 	/* caller already holds a reference, add for remainder */
430 	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
431 	data->nr_tags -= nr;
432 
433 	return rq_list_pop(data->cached_rq);
434 }
435 
__blk_mq_alloc_requests(struct blk_mq_alloc_data * data)436 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
437 {
438 	struct request_queue *q = data->q;
439 	u64 alloc_time_ns = 0;
440 	struct request *rq;
441 	unsigned int tag;
442 
443 	/* alloc_time includes depth and tag waits */
444 	if (blk_queue_rq_alloc_time(q))
445 		alloc_time_ns = ktime_get_ns();
446 
447 	if (data->cmd_flags & REQ_NOWAIT)
448 		data->flags |= BLK_MQ_REQ_NOWAIT;
449 
450 	if (q->elevator) {
451 		/*
452 		 * All requests use scheduler tags when an I/O scheduler is
453 		 * enabled for the queue.
454 		 */
455 		data->rq_flags |= RQF_SCHED_TAGS;
456 
457 		/*
458 		 * Flush/passthrough requests are special and go directly to the
459 		 * dispatch list.
460 		 */
461 		if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH &&
462 		    !blk_op_is_passthrough(data->cmd_flags)) {
463 			struct elevator_mq_ops *ops = &q->elevator->type->ops;
464 
465 			WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
466 
467 			data->rq_flags |= RQF_USE_SCHED;
468 			if (ops->limit_depth)
469 				ops->limit_depth(data->cmd_flags, data);
470 		}
471 	}
472 
473 retry:
474 	data->ctx = blk_mq_get_ctx(q);
475 	data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
476 	if (!(data->rq_flags & RQF_SCHED_TAGS))
477 		blk_mq_tag_busy(data->hctx);
478 
479 	if (data->flags & BLK_MQ_REQ_RESERVED)
480 		data->rq_flags |= RQF_RESV;
481 
482 	/*
483 	 * Try batched alloc if we want more than 1 tag.
484 	 */
485 	if (data->nr_tags > 1) {
486 		rq = __blk_mq_alloc_requests_batch(data);
487 		if (rq) {
488 			blk_mq_rq_time_init(rq, alloc_time_ns);
489 			return rq;
490 		}
491 		data->nr_tags = 1;
492 	}
493 
494 	/*
495 	 * Waiting allocations only fail because of an inactive hctx.  In that
496 	 * case just retry the hctx assignment and tag allocation as CPU hotplug
497 	 * should have migrated us to an online CPU by now.
498 	 */
499 	tag = blk_mq_get_tag(data);
500 	if (tag == BLK_MQ_NO_TAG) {
501 		if (data->flags & BLK_MQ_REQ_NOWAIT)
502 			return NULL;
503 		/*
504 		 * Give up the CPU and sleep for a random short time to
505 		 * ensure that thread using a realtime scheduling class
506 		 * are migrated off the CPU, and thus off the hctx that
507 		 * is going away.
508 		 */
509 		msleep(3);
510 		goto retry;
511 	}
512 
513 	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
514 	blk_mq_rq_time_init(rq, alloc_time_ns);
515 	return rq;
516 }
517 
blk_mq_rq_cache_fill(struct request_queue * q,struct blk_plug * plug,blk_opf_t opf,blk_mq_req_flags_t flags)518 static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
519 					    struct blk_plug *plug,
520 					    blk_opf_t opf,
521 					    blk_mq_req_flags_t flags)
522 {
523 	struct blk_mq_alloc_data data = {
524 		.q		= q,
525 		.flags		= flags,
526 		.cmd_flags	= opf,
527 		.nr_tags	= plug->nr_ios,
528 		.cached_rq	= &plug->cached_rq,
529 	};
530 	struct request *rq;
531 
532 	if (blk_queue_enter(q, flags))
533 		return NULL;
534 
535 	plug->nr_ios = 1;
536 
537 	rq = __blk_mq_alloc_requests(&data);
538 	if (unlikely(!rq))
539 		blk_queue_exit(q);
540 	return rq;
541 }
542 
blk_mq_alloc_cached_request(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags)543 static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
544 						   blk_opf_t opf,
545 						   blk_mq_req_flags_t flags)
546 {
547 	struct blk_plug *plug = current->plug;
548 	struct request *rq;
549 
550 	if (!plug)
551 		return NULL;
552 
553 	if (rq_list_empty(plug->cached_rq)) {
554 		if (plug->nr_ios == 1)
555 			return NULL;
556 		rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
557 		if (!rq)
558 			return NULL;
559 	} else {
560 		rq = rq_list_peek(&plug->cached_rq);
561 		if (!rq || rq->q != q)
562 			return NULL;
563 
564 		if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
565 			return NULL;
566 		if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
567 			return NULL;
568 
569 		plug->cached_rq = rq_list_next(rq);
570 		blk_mq_rq_time_init(rq, 0);
571 	}
572 
573 	rq->cmd_flags = opf;
574 	INIT_LIST_HEAD(&rq->queuelist);
575 	return rq;
576 }
577 
blk_mq_alloc_request(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags)578 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
579 		blk_mq_req_flags_t flags)
580 {
581 	struct request *rq;
582 
583 	rq = blk_mq_alloc_cached_request(q, opf, flags);
584 	if (!rq) {
585 		struct blk_mq_alloc_data data = {
586 			.q		= q,
587 			.flags		= flags,
588 			.cmd_flags	= opf,
589 			.nr_tags	= 1,
590 		};
591 		int ret;
592 
593 		ret = blk_queue_enter(q, flags);
594 		if (ret)
595 			return ERR_PTR(ret);
596 
597 		rq = __blk_mq_alloc_requests(&data);
598 		if (!rq)
599 			goto out_queue_exit;
600 	}
601 	rq->__data_len = 0;
602 	rq->__sector = (sector_t) -1;
603 	rq->bio = rq->biotail = NULL;
604 	return rq;
605 out_queue_exit:
606 	blk_queue_exit(q);
607 	return ERR_PTR(-EWOULDBLOCK);
608 }
609 EXPORT_SYMBOL(blk_mq_alloc_request);
610 
blk_mq_alloc_request_hctx(struct request_queue * q,blk_opf_t opf,blk_mq_req_flags_t flags,unsigned int hctx_idx)611 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
612 	blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
613 {
614 	struct blk_mq_alloc_data data = {
615 		.q		= q,
616 		.flags		= flags,
617 		.cmd_flags	= opf,
618 		.nr_tags	= 1,
619 	};
620 	u64 alloc_time_ns = 0;
621 	struct request *rq;
622 	unsigned int cpu;
623 	unsigned int tag;
624 	int ret;
625 
626 	/* alloc_time includes depth and tag waits */
627 	if (blk_queue_rq_alloc_time(q))
628 		alloc_time_ns = ktime_get_ns();
629 
630 	/*
631 	 * If the tag allocator sleeps we could get an allocation for a
632 	 * different hardware context.  No need to complicate the low level
633 	 * allocator for this for the rare use case of a command tied to
634 	 * a specific queue.
635 	 */
636 	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
637 	    WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
638 		return ERR_PTR(-EINVAL);
639 
640 	if (hctx_idx >= q->nr_hw_queues)
641 		return ERR_PTR(-EIO);
642 
643 	ret = blk_queue_enter(q, flags);
644 	if (ret)
645 		return ERR_PTR(ret);
646 
647 	/*
648 	 * Check if the hardware context is actually mapped to anything.
649 	 * If not tell the caller that it should skip this queue.
650 	 */
651 	ret = -EXDEV;
652 	data.hctx = xa_load(&q->hctx_table, hctx_idx);
653 	if (!blk_mq_hw_queue_mapped(data.hctx))
654 		goto out_queue_exit;
655 	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
656 	if (cpu >= nr_cpu_ids)
657 		goto out_queue_exit;
658 	data.ctx = __blk_mq_get_ctx(q, cpu);
659 
660 	if (q->elevator)
661 		data.rq_flags |= RQF_SCHED_TAGS;
662 	else
663 		blk_mq_tag_busy(data.hctx);
664 
665 	if (flags & BLK_MQ_REQ_RESERVED)
666 		data.rq_flags |= RQF_RESV;
667 
668 	ret = -EWOULDBLOCK;
669 	tag = blk_mq_get_tag(&data);
670 	if (tag == BLK_MQ_NO_TAG)
671 		goto out_queue_exit;
672 	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
673 	blk_mq_rq_time_init(rq, alloc_time_ns);
674 	rq->__data_len = 0;
675 	rq->__sector = (sector_t) -1;
676 	rq->bio = rq->biotail = NULL;
677 	return rq;
678 
679 out_queue_exit:
680 	blk_queue_exit(q);
681 	return ERR_PTR(ret);
682 }
683 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
684 
blk_mq_finish_request(struct request * rq)685 static void blk_mq_finish_request(struct request *rq)
686 {
687 	struct request_queue *q = rq->q;
688 
689 	if (rq->rq_flags & RQF_USE_SCHED) {
690 		q->elevator->type->ops.finish_request(rq);
691 		/*
692 		 * For postflush request that may need to be
693 		 * completed twice, we should clear this flag
694 		 * to avoid double finish_request() on the rq.
695 		 */
696 		rq->rq_flags &= ~RQF_USE_SCHED;
697 	}
698 }
699 
__blk_mq_free_request(struct request * rq)700 static void __blk_mq_free_request(struct request *rq)
701 {
702 	struct request_queue *q = rq->q;
703 	struct blk_mq_ctx *ctx = rq->mq_ctx;
704 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
705 	const int sched_tag = rq->internal_tag;
706 
707 	blk_crypto_free_request(rq);
708 	blk_pm_mark_last_busy(rq);
709 	rq->mq_hctx = NULL;
710 
711 	if (rq->rq_flags & RQF_MQ_INFLIGHT)
712 		__blk_mq_dec_active_requests(hctx);
713 
714 	if (rq->tag != BLK_MQ_NO_TAG)
715 		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
716 	if (sched_tag != BLK_MQ_NO_TAG)
717 		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
718 	blk_mq_sched_restart(hctx);
719 	blk_queue_exit(q);
720 }
721 
blk_mq_free_request(struct request * rq)722 void blk_mq_free_request(struct request *rq)
723 {
724 	struct request_queue *q = rq->q;
725 
726 	blk_mq_finish_request(rq);
727 
728 	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
729 		laptop_io_completion(q->disk->bdi);
730 
731 	rq_qos_done(q, rq);
732 
733 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
734 	if (req_ref_put_and_test(rq))
735 		__blk_mq_free_request(rq);
736 }
737 EXPORT_SYMBOL_GPL(blk_mq_free_request);
738 
blk_mq_free_plug_rqs(struct blk_plug * plug)739 void blk_mq_free_plug_rqs(struct blk_plug *plug)
740 {
741 	struct request *rq;
742 
743 	while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)
744 		blk_mq_free_request(rq);
745 }
746 
blk_dump_rq_flags(struct request * rq,char * msg)747 void blk_dump_rq_flags(struct request *rq, char *msg)
748 {
749 	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
750 		rq->q->disk ? rq->q->disk->disk_name : "?",
751 		(__force unsigned long long) rq->cmd_flags);
752 
753 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
754 	       (unsigned long long)blk_rq_pos(rq),
755 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
756 	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
757 	       rq->bio, rq->biotail, blk_rq_bytes(rq));
758 }
759 EXPORT_SYMBOL(blk_dump_rq_flags);
760 
req_bio_endio(struct request * rq,struct bio * bio,unsigned int nbytes,blk_status_t error)761 static void req_bio_endio(struct request *rq, struct bio *bio,
762 			  unsigned int nbytes, blk_status_t error)
763 {
764 	if (unlikely(error)) {
765 		bio->bi_status = error;
766 	} else if (req_op(rq) == REQ_OP_ZONE_APPEND) {
767 		/*
768 		 * Partial zone append completions cannot be supported as the
769 		 * BIO fragments may end up not being written sequentially.
770 		 * For such case, force the completed nbytes to be equal to
771 		 * the BIO size so that bio_advance() sets the BIO remaining
772 		 * size to 0 and we end up calling bio_endio() before returning.
773 		 */
774 		if (bio->bi_iter.bi_size != nbytes) {
775 			bio->bi_status = BLK_STS_IOERR;
776 			nbytes = bio->bi_iter.bi_size;
777 		} else {
778 			bio->bi_iter.bi_sector = rq->__sector;
779 		}
780 	}
781 
782 	bio_advance(bio, nbytes);
783 
784 	if (unlikely(rq->rq_flags & RQF_QUIET))
785 		bio_set_flag(bio, BIO_QUIET);
786 	/* don't actually finish bio if it's part of flush sequence */
787 	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
788 		bio_endio(bio);
789 }
790 
blk_account_io_completion(struct request * req,unsigned int bytes)791 static void blk_account_io_completion(struct request *req, unsigned int bytes)
792 {
793 	if (req->part && blk_do_io_stat(req)) {
794 		const int sgrp = op_stat_group(req_op(req));
795 
796 		part_stat_lock();
797 		part_stat_add(req->part, sectors[sgrp], bytes >> 9);
798 		part_stat_unlock();
799 	}
800 }
801 
blk_print_req_error(struct request * req,blk_status_t status)802 static void blk_print_req_error(struct request *req, blk_status_t status)
803 {
804 	printk_ratelimited(KERN_ERR
805 		"%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
806 		"phys_seg %u prio class %u\n",
807 		blk_status_to_str(status),
808 		req->q->disk ? req->q->disk->disk_name : "?",
809 		blk_rq_pos(req), (__force u32)req_op(req),
810 		blk_op_str(req_op(req)),
811 		(__force u32)(req->cmd_flags & ~REQ_OP_MASK),
812 		req->nr_phys_segments,
813 		IOPRIO_PRIO_CLASS(req->ioprio));
814 }
815 
816 /*
817  * Fully end IO on a request. Does not support partial completions, or
818  * errors.
819  */
blk_complete_request(struct request * req)820 static void blk_complete_request(struct request *req)
821 {
822 	const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
823 	int total_bytes = blk_rq_bytes(req);
824 	struct bio *bio = req->bio;
825 
826 	trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
827 
828 	if (!bio)
829 		return;
830 
831 #ifdef CONFIG_BLK_DEV_INTEGRITY
832 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
833 		req->q->integrity.profile->complete_fn(req, total_bytes);
834 #endif
835 
836 	/*
837 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
838 	 * bio_endio().  Therefore, the keyslot must be released before that.
839 	 */
840 	blk_crypto_rq_put_keyslot(req);
841 
842 	blk_account_io_completion(req, total_bytes);
843 
844 	do {
845 		struct bio *next = bio->bi_next;
846 
847 		/* Completion has already been traced */
848 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
849 
850 		if (req_op(req) == REQ_OP_ZONE_APPEND)
851 			bio->bi_iter.bi_sector = req->__sector;
852 
853 		if (!is_flush)
854 			bio_endio(bio);
855 		bio = next;
856 	} while (bio);
857 
858 	/*
859 	 * Reset counters so that the request stacking driver
860 	 * can find how many bytes remain in the request
861 	 * later.
862 	 */
863 	if (!req->end_io) {
864 		req->bio = NULL;
865 		req->__data_len = 0;
866 	}
867 }
868 
869 /**
870  * blk_update_request - Complete multiple bytes without completing the request
871  * @req:      the request being processed
872  * @error:    block status code
873  * @nr_bytes: number of bytes to complete for @req
874  *
875  * Description:
876  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
877  *     the request structure even if @req doesn't have leftover.
878  *     If @req has leftover, sets it up for the next range of segments.
879  *
880  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
881  *     %false return from this function.
882  *
883  * Note:
884  *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
885  *      except in the consistency check at the end of this function.
886  *
887  * Return:
888  *     %false - this request doesn't have any more data
889  *     %true  - this request has more data
890  **/
blk_update_request(struct request * req,blk_status_t error,unsigned int nr_bytes)891 bool blk_update_request(struct request *req, blk_status_t error,
892 		unsigned int nr_bytes)
893 {
894 	int total_bytes;
895 
896 	trace_block_rq_complete(req, error, nr_bytes);
897 
898 	if (!req->bio)
899 		return false;
900 
901 #ifdef CONFIG_BLK_DEV_INTEGRITY
902 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
903 	    error == BLK_STS_OK)
904 		req->q->integrity.profile->complete_fn(req, nr_bytes);
905 #endif
906 
907 	/*
908 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
909 	 * bio_endio().  Therefore, the keyslot must be released before that.
910 	 */
911 	if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
912 		__blk_crypto_rq_put_keyslot(req);
913 
914 	if (unlikely(error && !blk_rq_is_passthrough(req) &&
915 		     !(req->rq_flags & RQF_QUIET)) &&
916 		     !test_bit(GD_DEAD, &req->q->disk->state)) {
917 		blk_print_req_error(req, error);
918 		trace_block_rq_error(req, error, nr_bytes);
919 	}
920 
921 	blk_account_io_completion(req, nr_bytes);
922 
923 	total_bytes = 0;
924 	while (req->bio) {
925 		struct bio *bio = req->bio;
926 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
927 
928 		if (bio_bytes == bio->bi_iter.bi_size)
929 			req->bio = bio->bi_next;
930 
931 		/* Completion has already been traced */
932 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
933 		req_bio_endio(req, bio, bio_bytes, error);
934 
935 		total_bytes += bio_bytes;
936 		nr_bytes -= bio_bytes;
937 
938 		if (!nr_bytes)
939 			break;
940 	}
941 
942 	/*
943 	 * completely done
944 	 */
945 	if (!req->bio) {
946 		/*
947 		 * Reset counters so that the request stacking driver
948 		 * can find how many bytes remain in the request
949 		 * later.
950 		 */
951 		req->__data_len = 0;
952 		return false;
953 	}
954 
955 	req->__data_len -= total_bytes;
956 
957 	/* update sector only for requests with clear definition of sector */
958 	if (!blk_rq_is_passthrough(req))
959 		req->__sector += total_bytes >> 9;
960 
961 	/* mixed attributes always follow the first bio */
962 	if (req->rq_flags & RQF_MIXED_MERGE) {
963 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
964 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
965 	}
966 
967 	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
968 		/*
969 		 * If total number of sectors is less than the first segment
970 		 * size, something has gone terribly wrong.
971 		 */
972 		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
973 			blk_dump_rq_flags(req, "request botched");
974 			req->__data_len = blk_rq_cur_bytes(req);
975 		}
976 
977 		/* recalculate the number of segments */
978 		req->nr_phys_segments = blk_recalc_rq_segments(req);
979 	}
980 
981 	return true;
982 }
983 EXPORT_SYMBOL_GPL(blk_update_request);
984 
blk_account_io_done(struct request * req,u64 now)985 static inline void blk_account_io_done(struct request *req, u64 now)
986 {
987 	trace_block_io_done(req);
988 
989 	/*
990 	 * Account IO completion.  flush_rq isn't accounted as a
991 	 * normal IO on queueing nor completion.  Accounting the
992 	 * containing request is enough.
993 	 */
994 	if (blk_do_io_stat(req) && req->part &&
995 	    !(req->rq_flags & RQF_FLUSH_SEQ)) {
996 		const int sgrp = op_stat_group(req_op(req));
997 
998 		part_stat_lock();
999 		update_io_ticks(req->part, jiffies, true);
1000 		part_stat_inc(req->part, ios[sgrp]);
1001 		part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
1002 		part_stat_unlock();
1003 	}
1004 }
1005 
blk_account_io_start(struct request * req)1006 static inline void blk_account_io_start(struct request *req)
1007 {
1008 	trace_block_io_start(req);
1009 
1010 	if (blk_do_io_stat(req)) {
1011 		/*
1012 		 * All non-passthrough requests are created from a bio with one
1013 		 * exception: when a flush command that is part of a flush sequence
1014 		 * generated by the state machine in blk-flush.c is cloned onto the
1015 		 * lower device by dm-multipath we can get here without a bio.
1016 		 */
1017 		if (req->bio)
1018 			req->part = req->bio->bi_bdev;
1019 		else
1020 			req->part = req->q->disk->part0;
1021 
1022 		part_stat_lock();
1023 		update_io_ticks(req->part, jiffies, false);
1024 		part_stat_unlock();
1025 	}
1026 }
1027 
__blk_mq_end_request_acct(struct request * rq,u64 now)1028 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
1029 {
1030 	if (rq->rq_flags & RQF_STATS)
1031 		blk_stat_add(rq, now);
1032 
1033 	blk_mq_sched_completed_request(rq, now);
1034 	blk_account_io_done(rq, now);
1035 }
1036 
__blk_mq_end_request(struct request * rq,blk_status_t error)1037 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
1038 {
1039 	if (blk_mq_need_time_stamp(rq))
1040 		__blk_mq_end_request_acct(rq, ktime_get_ns());
1041 
1042 	blk_mq_finish_request(rq);
1043 
1044 	if (rq->end_io) {
1045 		rq_qos_done(rq->q, rq);
1046 		if (rq->end_io(rq, error) == RQ_END_IO_FREE)
1047 			blk_mq_free_request(rq);
1048 	} else {
1049 		blk_mq_free_request(rq);
1050 	}
1051 }
1052 EXPORT_SYMBOL(__blk_mq_end_request);
1053 
blk_mq_end_request(struct request * rq,blk_status_t error)1054 void blk_mq_end_request(struct request *rq, blk_status_t error)
1055 {
1056 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
1057 		BUG();
1058 	__blk_mq_end_request(rq, error);
1059 }
1060 EXPORT_SYMBOL(blk_mq_end_request);
1061 
1062 #define TAG_COMP_BATCH		32
1063 
blk_mq_flush_tag_batch(struct blk_mq_hw_ctx * hctx,int * tag_array,int nr_tags)1064 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
1065 					  int *tag_array, int nr_tags)
1066 {
1067 	struct request_queue *q = hctx->queue;
1068 
1069 	/*
1070 	 * All requests should have been marked as RQF_MQ_INFLIGHT, so
1071 	 * update hctx->nr_active in batch
1072 	 */
1073 	if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
1074 		__blk_mq_sub_active_requests(hctx, nr_tags);
1075 
1076 	blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
1077 	percpu_ref_put_many(&q->q_usage_counter, nr_tags);
1078 }
1079 
blk_mq_end_request_batch(struct io_comp_batch * iob)1080 void blk_mq_end_request_batch(struct io_comp_batch *iob)
1081 {
1082 	int tags[TAG_COMP_BATCH], nr_tags = 0;
1083 	struct blk_mq_hw_ctx *cur_hctx = NULL;
1084 	struct request *rq;
1085 	u64 now = 0;
1086 
1087 	if (iob->need_ts)
1088 		now = ktime_get_ns();
1089 
1090 	while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
1091 		prefetch(rq->bio);
1092 		prefetch(rq->rq_next);
1093 
1094 		blk_complete_request(rq);
1095 		if (iob->need_ts)
1096 			__blk_mq_end_request_acct(rq, now);
1097 
1098 		blk_mq_finish_request(rq);
1099 
1100 		rq_qos_done(rq->q, rq);
1101 
1102 		/*
1103 		 * If end_io handler returns NONE, then it still has
1104 		 * ownership of the request.
1105 		 */
1106 		if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
1107 			continue;
1108 
1109 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1110 		if (!req_ref_put_and_test(rq))
1111 			continue;
1112 
1113 		blk_crypto_free_request(rq);
1114 		blk_pm_mark_last_busy(rq);
1115 
1116 		if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
1117 			if (cur_hctx)
1118 				blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1119 			nr_tags = 0;
1120 			cur_hctx = rq->mq_hctx;
1121 		}
1122 		tags[nr_tags++] = rq->tag;
1123 	}
1124 
1125 	if (nr_tags)
1126 		blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1127 }
1128 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
1129 
blk_complete_reqs(struct llist_head * list)1130 static void blk_complete_reqs(struct llist_head *list)
1131 {
1132 	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
1133 	struct request *rq, *next;
1134 
1135 	llist_for_each_entry_safe(rq, next, entry, ipi_list)
1136 		rq->q->mq_ops->complete(rq);
1137 }
1138 
blk_done_softirq(struct softirq_action * h)1139 static __latent_entropy void blk_done_softirq(struct softirq_action *h)
1140 {
1141 	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
1142 }
1143 
blk_softirq_cpu_dead(unsigned int cpu)1144 static int blk_softirq_cpu_dead(unsigned int cpu)
1145 {
1146 	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
1147 	return 0;
1148 }
1149 
__blk_mq_complete_request_remote(void * data)1150 static void __blk_mq_complete_request_remote(void *data)
1151 {
1152 	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
1153 }
1154 
blk_mq_complete_need_ipi(struct request * rq)1155 static inline bool blk_mq_complete_need_ipi(struct request *rq)
1156 {
1157 	int cpu = raw_smp_processor_id();
1158 
1159 	if (!IS_ENABLED(CONFIG_SMP) ||
1160 	    !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
1161 		return false;
1162 	/*
1163 	 * With force threaded interrupts enabled, raising softirq from an SMP
1164 	 * function call will always result in waking the ksoftirqd thread.
1165 	 * This is probably worse than completing the request on a different
1166 	 * cache domain.
1167 	 */
1168 	if (force_irqthreads())
1169 		return false;
1170 
1171 	/* same CPU or cache domain?  Complete locally */
1172 	if (cpu == rq->mq_ctx->cpu ||
1173 	    (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
1174 	     cpus_share_cache(cpu, rq->mq_ctx->cpu)))
1175 		return false;
1176 
1177 	/* don't try to IPI to an offline CPU */
1178 	return cpu_online(rq->mq_ctx->cpu);
1179 }
1180 
blk_mq_complete_send_ipi(struct request * rq)1181 static void blk_mq_complete_send_ipi(struct request *rq)
1182 {
1183 	unsigned int cpu;
1184 
1185 	cpu = rq->mq_ctx->cpu;
1186 	if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
1187 		smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
1188 }
1189 
blk_mq_raise_softirq(struct request * rq)1190 static void blk_mq_raise_softirq(struct request *rq)
1191 {
1192 	struct llist_head *list;
1193 
1194 	preempt_disable();
1195 	list = this_cpu_ptr(&blk_cpu_done);
1196 	if (llist_add(&rq->ipi_list, list))
1197 		raise_softirq(BLOCK_SOFTIRQ);
1198 	preempt_enable();
1199 }
1200 
blk_mq_complete_request_remote(struct request * rq)1201 bool blk_mq_complete_request_remote(struct request *rq)
1202 {
1203 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
1204 
1205 	/*
1206 	 * For request which hctx has only one ctx mapping,
1207 	 * or a polled request, always complete locally,
1208 	 * it's pointless to redirect the completion.
1209 	 */
1210 	if ((rq->mq_hctx->nr_ctx == 1 &&
1211 	     rq->mq_ctx->cpu == raw_smp_processor_id()) ||
1212 	     rq->cmd_flags & REQ_POLLED)
1213 		return false;
1214 
1215 	if (blk_mq_complete_need_ipi(rq)) {
1216 		blk_mq_complete_send_ipi(rq);
1217 		return true;
1218 	}
1219 
1220 	if (rq->q->nr_hw_queues == 1) {
1221 		blk_mq_raise_softirq(rq);
1222 		return true;
1223 	}
1224 	return false;
1225 }
1226 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
1227 
1228 /**
1229  * blk_mq_complete_request - end I/O on a request
1230  * @rq:		the request being processed
1231  *
1232  * Description:
1233  *	Complete a request by scheduling the ->complete_rq operation.
1234  **/
blk_mq_complete_request(struct request * rq)1235 void blk_mq_complete_request(struct request *rq)
1236 {
1237 	if (!blk_mq_complete_request_remote(rq))
1238 		rq->q->mq_ops->complete(rq);
1239 }
1240 EXPORT_SYMBOL(blk_mq_complete_request);
1241 
1242 /**
1243  * blk_mq_start_request - Start processing a request
1244  * @rq: Pointer to request to be started
1245  *
1246  * Function used by device drivers to notify the block layer that a request
1247  * is going to be processed now, so blk layer can do proper initializations
1248  * such as starting the timeout timer.
1249  */
blk_mq_start_request(struct request * rq)1250 void blk_mq_start_request(struct request *rq)
1251 {
1252 	struct request_queue *q = rq->q;
1253 
1254 	trace_block_rq_issue(rq);
1255 
1256 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
1257 		rq->io_start_time_ns = ktime_get_ns();
1258 		rq->stats_sectors = blk_rq_sectors(rq);
1259 		rq->rq_flags |= RQF_STATS;
1260 		rq_qos_issue(q, rq);
1261 	}
1262 
1263 	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
1264 
1265 	blk_add_timer(rq);
1266 	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
1267 
1268 #ifdef CONFIG_BLK_DEV_INTEGRITY
1269 	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
1270 		q->integrity.profile->prepare_fn(rq);
1271 #endif
1272 	if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
1273 	        WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
1274 }
1275 EXPORT_SYMBOL(blk_mq_start_request);
1276 
1277 /*
1278  * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
1279  * queues. This is important for md arrays to benefit from merging
1280  * requests.
1281  */
blk_plug_max_rq_count(struct blk_plug * plug)1282 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
1283 {
1284 	if (plug->multiple_queues)
1285 		return BLK_MAX_REQUEST_COUNT * 2;
1286 	return BLK_MAX_REQUEST_COUNT;
1287 }
1288 
blk_add_rq_to_plug(struct blk_plug * plug,struct request * rq)1289 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
1290 {
1291 	struct request *last = rq_list_peek(&plug->mq_list);
1292 
1293 	if (!plug->rq_count) {
1294 		trace_block_plug(rq->q);
1295 	} else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
1296 		   (!blk_queue_nomerges(rq->q) &&
1297 		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1298 		blk_mq_flush_plug_list(plug, false);
1299 		last = NULL;
1300 		trace_block_plug(rq->q);
1301 	}
1302 
1303 	if (!plug->multiple_queues && last && last->q != rq->q)
1304 		plug->multiple_queues = true;
1305 	/*
1306 	 * Any request allocated from sched tags can't be issued to
1307 	 * ->queue_rqs() directly
1308 	 */
1309 	if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1310 		plug->has_elevator = true;
1311 	rq->rq_next = NULL;
1312 	rq_list_add(&plug->mq_list, rq);
1313 	plug->rq_count++;
1314 }
1315 
1316 /**
1317  * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
1318  * @rq:		request to insert
1319  * @at_head:    insert request at head or tail of queue
1320  *
1321  * Description:
1322  *    Insert a fully prepared request at the back of the I/O scheduler queue
1323  *    for execution.  Don't wait for completion.
1324  *
1325  * Note:
1326  *    This function will invoke @done directly if the queue is dead.
1327  */
blk_execute_rq_nowait(struct request * rq,bool at_head)1328 void blk_execute_rq_nowait(struct request *rq, bool at_head)
1329 {
1330 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1331 
1332 	WARN_ON(irqs_disabled());
1333 	WARN_ON(!blk_rq_is_passthrough(rq));
1334 
1335 	blk_account_io_start(rq);
1336 
1337 	/*
1338 	 * As plugging can be enabled for passthrough requests on a zoned
1339 	 * device, directly accessing the plug instead of using blk_mq_plug()
1340 	 * should not have any consequences.
1341 	 */
1342 	if (current->plug && !at_head) {
1343 		blk_add_rq_to_plug(current->plug, rq);
1344 		return;
1345 	}
1346 
1347 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1348 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
1349 }
1350 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
1351 
1352 struct blk_rq_wait {
1353 	struct completion done;
1354 	blk_status_t ret;
1355 };
1356 
blk_end_sync_rq(struct request * rq,blk_status_t ret)1357 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret)
1358 {
1359 	struct blk_rq_wait *wait = rq->end_io_data;
1360 
1361 	wait->ret = ret;
1362 	complete(&wait->done);
1363 	return RQ_END_IO_NONE;
1364 }
1365 
blk_rq_is_poll(struct request * rq)1366 bool blk_rq_is_poll(struct request *rq)
1367 {
1368 	if (!rq->mq_hctx)
1369 		return false;
1370 	if (rq->mq_hctx->type != HCTX_TYPE_POLL)
1371 		return false;
1372 	return true;
1373 }
1374 EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1375 
blk_rq_poll_completion(struct request * rq,struct completion * wait)1376 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
1377 {
1378 	do {
1379 		blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0);
1380 		cond_resched();
1381 	} while (!completion_done(wait));
1382 }
1383 
1384 /**
1385  * blk_execute_rq - insert a request into queue for execution
1386  * @rq:		request to insert
1387  * @at_head:    insert request at head or tail of queue
1388  *
1389  * Description:
1390  *    Insert a fully prepared request at the back of the I/O scheduler queue
1391  *    for execution and wait for completion.
1392  * Return: The blk_status_t result provided to blk_mq_end_request().
1393  */
blk_execute_rq(struct request * rq,bool at_head)1394 blk_status_t blk_execute_rq(struct request *rq, bool at_head)
1395 {
1396 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1397 	struct blk_rq_wait wait = {
1398 		.done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
1399 	};
1400 
1401 	WARN_ON(irqs_disabled());
1402 	WARN_ON(!blk_rq_is_passthrough(rq));
1403 
1404 	rq->end_io_data = &wait;
1405 	rq->end_io = blk_end_sync_rq;
1406 
1407 	blk_account_io_start(rq);
1408 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1409 	blk_mq_run_hw_queue(hctx, false);
1410 
1411 	if (blk_rq_is_poll(rq)) {
1412 		blk_rq_poll_completion(rq, &wait.done);
1413 	} else {
1414 		/*
1415 		 * Prevent hang_check timer from firing at us during very long
1416 		 * I/O
1417 		 */
1418 		unsigned long hang_check = sysctl_hung_task_timeout_secs;
1419 
1420 		if (hang_check)
1421 			while (!wait_for_completion_io_timeout(&wait.done,
1422 					hang_check * (HZ/2)))
1423 				;
1424 		else
1425 			wait_for_completion_io(&wait.done);
1426 	}
1427 
1428 	return wait.ret;
1429 }
1430 EXPORT_SYMBOL(blk_execute_rq);
1431 
__blk_mq_requeue_request(struct request * rq)1432 static void __blk_mq_requeue_request(struct request *rq)
1433 {
1434 	struct request_queue *q = rq->q;
1435 
1436 	blk_mq_put_driver_tag(rq);
1437 
1438 	trace_block_rq_requeue(rq);
1439 	rq_qos_requeue(q, rq);
1440 
1441 	if (blk_mq_request_started(rq)) {
1442 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1443 		rq->rq_flags &= ~RQF_TIMED_OUT;
1444 	}
1445 }
1446 
blk_mq_requeue_request(struct request * rq,bool kick_requeue_list)1447 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
1448 {
1449 	struct request_queue *q = rq->q;
1450 	unsigned long flags;
1451 
1452 	__blk_mq_requeue_request(rq);
1453 
1454 	/* this request will be re-inserted to io scheduler queue */
1455 	blk_mq_sched_requeue_request(rq);
1456 
1457 	spin_lock_irqsave(&q->requeue_lock, flags);
1458 	list_add_tail(&rq->queuelist, &q->requeue_list);
1459 	spin_unlock_irqrestore(&q->requeue_lock, flags);
1460 
1461 	if (kick_requeue_list)
1462 		blk_mq_kick_requeue_list(q);
1463 }
1464 EXPORT_SYMBOL(blk_mq_requeue_request);
1465 
blk_mq_requeue_work(struct work_struct * work)1466 static void blk_mq_requeue_work(struct work_struct *work)
1467 {
1468 	struct request_queue *q =
1469 		container_of(work, struct request_queue, requeue_work.work);
1470 	LIST_HEAD(rq_list);
1471 	LIST_HEAD(flush_list);
1472 	struct request *rq;
1473 
1474 	spin_lock_irq(&q->requeue_lock);
1475 	list_splice_init(&q->requeue_list, &rq_list);
1476 	list_splice_init(&q->flush_list, &flush_list);
1477 	spin_unlock_irq(&q->requeue_lock);
1478 
1479 	while (!list_empty(&rq_list)) {
1480 		rq = list_entry(rq_list.next, struct request, queuelist);
1481 		/*
1482 		 * If RQF_DONTPREP ist set, the request has been started by the
1483 		 * driver already and might have driver-specific data allocated
1484 		 * already.  Insert it into the hctx dispatch list to avoid
1485 		 * block layer merges for the request.
1486 		 */
1487 		if (rq->rq_flags & RQF_DONTPREP) {
1488 			list_del_init(&rq->queuelist);
1489 			blk_mq_request_bypass_insert(rq, 0);
1490 		} else {
1491 			list_del_init(&rq->queuelist);
1492 			blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
1493 		}
1494 	}
1495 
1496 	while (!list_empty(&flush_list)) {
1497 		rq = list_entry(flush_list.next, struct request, queuelist);
1498 		list_del_init(&rq->queuelist);
1499 		blk_mq_insert_request(rq, 0);
1500 	}
1501 
1502 	blk_mq_run_hw_queues(q, false);
1503 }
1504 
blk_mq_kick_requeue_list(struct request_queue * q)1505 void blk_mq_kick_requeue_list(struct request_queue *q)
1506 {
1507 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
1508 }
1509 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
1510 
blk_mq_delay_kick_requeue_list(struct request_queue * q,unsigned long msecs)1511 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
1512 				    unsigned long msecs)
1513 {
1514 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
1515 				    msecs_to_jiffies(msecs));
1516 }
1517 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
1518 
blk_is_flush_data_rq(struct request * rq)1519 static bool blk_is_flush_data_rq(struct request *rq)
1520 {
1521 	return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
1522 }
1523 
blk_mq_rq_inflight(struct request * rq,void * priv)1524 static bool blk_mq_rq_inflight(struct request *rq, void *priv)
1525 {
1526 	/*
1527 	 * If we find a request that isn't idle we know the queue is busy
1528 	 * as it's checked in the iter.
1529 	 * Return false to stop the iteration.
1530 	 *
1531 	 * In case of queue quiesce, if one flush data request is completed,
1532 	 * don't count it as inflight given the flush sequence is suspended,
1533 	 * and the original flush data request is invisible to driver, just
1534 	 * like other pending requests because of quiesce
1535 	 */
1536 	if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
1537 				blk_is_flush_data_rq(rq) &&
1538 				blk_mq_request_completed(rq))) {
1539 		bool *busy = priv;
1540 
1541 		*busy = true;
1542 		return false;
1543 	}
1544 
1545 	return true;
1546 }
1547 
blk_mq_queue_inflight(struct request_queue * q)1548 bool blk_mq_queue_inflight(struct request_queue *q)
1549 {
1550 	bool busy = false;
1551 
1552 	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
1553 	return busy;
1554 }
1555 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
1556 
blk_mq_rq_timed_out(struct request * req)1557 static void blk_mq_rq_timed_out(struct request *req)
1558 {
1559 	req->rq_flags |= RQF_TIMED_OUT;
1560 	if (req->q->mq_ops->timeout) {
1561 		enum blk_eh_timer_return ret;
1562 
1563 		ret = req->q->mq_ops->timeout(req);
1564 		if (ret == BLK_EH_DONE)
1565 			return;
1566 		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
1567 	}
1568 
1569 	blk_add_timer(req);
1570 }
1571 
1572 struct blk_expired_data {
1573 	bool has_timedout_rq;
1574 	unsigned long next;
1575 	unsigned long timeout_start;
1576 };
1577 
blk_mq_req_expired(struct request * rq,struct blk_expired_data * expired)1578 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
1579 {
1580 	unsigned long deadline;
1581 
1582 	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
1583 		return false;
1584 	if (rq->rq_flags & RQF_TIMED_OUT)
1585 		return false;
1586 
1587 	deadline = READ_ONCE(rq->deadline);
1588 	if (time_after_eq(expired->timeout_start, deadline))
1589 		return true;
1590 
1591 	if (expired->next == 0)
1592 		expired->next = deadline;
1593 	else if (time_after(expired->next, deadline))
1594 		expired->next = deadline;
1595 	return false;
1596 }
1597 
blk_mq_put_rq_ref(struct request * rq)1598 void blk_mq_put_rq_ref(struct request *rq)
1599 {
1600 	if (is_flush_rq(rq)) {
1601 		if (rq->end_io(rq, 0) == RQ_END_IO_FREE)
1602 			blk_mq_free_request(rq);
1603 	} else if (req_ref_put_and_test(rq)) {
1604 		__blk_mq_free_request(rq);
1605 	}
1606 }
1607 
blk_mq_check_expired(struct request * rq,void * priv)1608 static bool blk_mq_check_expired(struct request *rq, void *priv)
1609 {
1610 	struct blk_expired_data *expired = priv;
1611 
1612 	/*
1613 	 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1614 	 * be reallocated underneath the timeout handler's processing, then
1615 	 * the expire check is reliable. If the request is not expired, then
1616 	 * it was completed and reallocated as a new request after returning
1617 	 * from blk_mq_check_expired().
1618 	 */
1619 	if (blk_mq_req_expired(rq, expired)) {
1620 		expired->has_timedout_rq = true;
1621 		return false;
1622 	}
1623 	return true;
1624 }
1625 
blk_mq_handle_expired(struct request * rq,void * priv)1626 static bool blk_mq_handle_expired(struct request *rq, void *priv)
1627 {
1628 	struct blk_expired_data *expired = priv;
1629 
1630 	if (blk_mq_req_expired(rq, expired))
1631 		blk_mq_rq_timed_out(rq);
1632 	return true;
1633 }
1634 
blk_mq_timeout_work(struct work_struct * work)1635 static void blk_mq_timeout_work(struct work_struct *work)
1636 {
1637 	struct request_queue *q =
1638 		container_of(work, struct request_queue, timeout_work);
1639 	struct blk_expired_data expired = {
1640 		.timeout_start = jiffies,
1641 	};
1642 	struct blk_mq_hw_ctx *hctx;
1643 	unsigned long i;
1644 
1645 	/* A deadlock might occur if a request is stuck requiring a
1646 	 * timeout at the same time a queue freeze is waiting
1647 	 * completion, since the timeout code would not be able to
1648 	 * acquire the queue reference here.
1649 	 *
1650 	 * That's why we don't use blk_queue_enter here; instead, we use
1651 	 * percpu_ref_tryget directly, because we need to be able to
1652 	 * obtain a reference even in the short window between the queue
1653 	 * starting to freeze, by dropping the first reference in
1654 	 * blk_freeze_queue_start, and the moment the last request is
1655 	 * consumed, marked by the instant q_usage_counter reaches
1656 	 * zero.
1657 	 */
1658 	if (!percpu_ref_tryget(&q->q_usage_counter))
1659 		return;
1660 
1661 	/* check if there is any timed-out request */
1662 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
1663 	if (expired.has_timedout_rq) {
1664 		/*
1665 		 * Before walking tags, we must ensure any submit started
1666 		 * before the current time has finished. Since the submit
1667 		 * uses srcu or rcu, wait for a synchronization point to
1668 		 * ensure all running submits have finished
1669 		 */
1670 		blk_mq_wait_quiesce_done(q->tag_set);
1671 
1672 		expired.next = 0;
1673 		blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
1674 	}
1675 
1676 	if (expired.next != 0) {
1677 		mod_timer(&q->timeout, expired.next);
1678 	} else {
1679 		/*
1680 		 * Request timeouts are handled as a forward rolling timer. If
1681 		 * we end up here it means that no requests are pending and
1682 		 * also that no request has been pending for a while. Mark
1683 		 * each hctx as idle.
1684 		 */
1685 		queue_for_each_hw_ctx(q, hctx, i) {
1686 			/* the hctx may be unmapped, so check it here */
1687 			if (blk_mq_hw_queue_mapped(hctx))
1688 				blk_mq_tag_idle(hctx);
1689 		}
1690 	}
1691 	blk_queue_exit(q);
1692 }
1693 
1694 struct flush_busy_ctx_data {
1695 	struct blk_mq_hw_ctx *hctx;
1696 	struct list_head *list;
1697 };
1698 
flush_busy_ctx(struct sbitmap * sb,unsigned int bitnr,void * data)1699 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
1700 {
1701 	struct flush_busy_ctx_data *flush_data = data;
1702 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1703 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1704 	enum hctx_type type = hctx->type;
1705 
1706 	spin_lock(&ctx->lock);
1707 	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1708 	sbitmap_clear_bit(sb, bitnr);
1709 	spin_unlock(&ctx->lock);
1710 	return true;
1711 }
1712 
1713 /*
1714  * Process software queues that have been marked busy, splicing them
1715  * to the for-dispatch
1716  */
blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx * hctx,struct list_head * list)1717 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1718 {
1719 	struct flush_busy_ctx_data data = {
1720 		.hctx = hctx,
1721 		.list = list,
1722 	};
1723 
1724 	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1725 }
1726 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1727 
1728 struct dispatch_rq_data {
1729 	struct blk_mq_hw_ctx *hctx;
1730 	struct request *rq;
1731 };
1732 
dispatch_rq_from_ctx(struct sbitmap * sb,unsigned int bitnr,void * data)1733 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1734 		void *data)
1735 {
1736 	struct dispatch_rq_data *dispatch_data = data;
1737 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1738 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1739 	enum hctx_type type = hctx->type;
1740 
1741 	spin_lock(&ctx->lock);
1742 	if (!list_empty(&ctx->rq_lists[type])) {
1743 		dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1744 		list_del_init(&dispatch_data->rq->queuelist);
1745 		if (list_empty(&ctx->rq_lists[type]))
1746 			sbitmap_clear_bit(sb, bitnr);
1747 	}
1748 	spin_unlock(&ctx->lock);
1749 
1750 	return !dispatch_data->rq;
1751 }
1752 
blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * start)1753 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1754 					struct blk_mq_ctx *start)
1755 {
1756 	unsigned off = start ? start->index_hw[hctx->type] : 0;
1757 	struct dispatch_rq_data data = {
1758 		.hctx = hctx,
1759 		.rq   = NULL,
1760 	};
1761 
1762 	__sbitmap_for_each_set(&hctx->ctx_map, off,
1763 			       dispatch_rq_from_ctx, &data);
1764 
1765 	return data.rq;
1766 }
1767 
__blk_mq_alloc_driver_tag(struct request * rq)1768 static bool __blk_mq_alloc_driver_tag(struct request *rq)
1769 {
1770 	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1771 	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1772 	int tag;
1773 
1774 	blk_mq_tag_busy(rq->mq_hctx);
1775 
1776 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1777 		bt = &rq->mq_hctx->tags->breserved_tags;
1778 		tag_offset = 0;
1779 	} else {
1780 		if (!hctx_may_queue(rq->mq_hctx, bt))
1781 			return false;
1782 	}
1783 
1784 	tag = __sbitmap_queue_get(bt);
1785 	if (tag == BLK_MQ_NO_TAG)
1786 		return false;
1787 
1788 	rq->tag = tag + tag_offset;
1789 	return true;
1790 }
1791 
__blk_mq_get_driver_tag(struct blk_mq_hw_ctx * hctx,struct request * rq)1792 bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq)
1793 {
1794 	if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_alloc_driver_tag(rq))
1795 		return false;
1796 
1797 	if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1798 			!(rq->rq_flags & RQF_MQ_INFLIGHT)) {
1799 		rq->rq_flags |= RQF_MQ_INFLIGHT;
1800 		__blk_mq_inc_active_requests(hctx);
1801 	}
1802 	hctx->tags->rqs[rq->tag] = rq;
1803 	return true;
1804 }
1805 
blk_mq_dispatch_wake(wait_queue_entry_t * wait,unsigned mode,int flags,void * key)1806 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1807 				int flags, void *key)
1808 {
1809 	struct blk_mq_hw_ctx *hctx;
1810 
1811 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1812 
1813 	spin_lock(&hctx->dispatch_wait_lock);
1814 	if (!list_empty(&wait->entry)) {
1815 		struct sbitmap_queue *sbq;
1816 
1817 		list_del_init(&wait->entry);
1818 		sbq = &hctx->tags->bitmap_tags;
1819 		atomic_dec(&sbq->ws_active);
1820 	}
1821 	spin_unlock(&hctx->dispatch_wait_lock);
1822 
1823 	blk_mq_run_hw_queue(hctx, true);
1824 	return 1;
1825 }
1826 
1827 /*
1828  * Mark us waiting for a tag. For shared tags, this involves hooking us into
1829  * the tag wakeups. For non-shared tags, we can simply mark us needing a
1830  * restart. For both cases, take care to check the condition again after
1831  * marking us as waiting.
1832  */
blk_mq_mark_tag_wait(struct blk_mq_hw_ctx * hctx,struct request * rq)1833 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1834 				 struct request *rq)
1835 {
1836 	struct sbitmap_queue *sbq;
1837 	struct wait_queue_head *wq;
1838 	wait_queue_entry_t *wait;
1839 	bool ret;
1840 
1841 	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1842 	    !(blk_mq_is_shared_tags(hctx->flags))) {
1843 		blk_mq_sched_mark_restart_hctx(hctx);
1844 
1845 		/*
1846 		 * It's possible that a tag was freed in the window between the
1847 		 * allocation failure and adding the hardware queue to the wait
1848 		 * queue.
1849 		 *
1850 		 * Don't clear RESTART here, someone else could have set it.
1851 		 * At most this will cost an extra queue run.
1852 		 */
1853 		return blk_mq_get_driver_tag(rq);
1854 	}
1855 
1856 	wait = &hctx->dispatch_wait;
1857 	if (!list_empty_careful(&wait->entry))
1858 		return false;
1859 
1860 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
1861 		sbq = &hctx->tags->breserved_tags;
1862 	else
1863 		sbq = &hctx->tags->bitmap_tags;
1864 	wq = &bt_wait_ptr(sbq, hctx)->wait;
1865 
1866 	spin_lock_irq(&wq->lock);
1867 	spin_lock(&hctx->dispatch_wait_lock);
1868 	if (!list_empty(&wait->entry)) {
1869 		spin_unlock(&hctx->dispatch_wait_lock);
1870 		spin_unlock_irq(&wq->lock);
1871 		return false;
1872 	}
1873 
1874 	atomic_inc(&sbq->ws_active);
1875 	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1876 	__add_wait_queue(wq, wait);
1877 
1878 	/*
1879 	 * Add one explicit barrier since blk_mq_get_driver_tag() may
1880 	 * not imply barrier in case of failure.
1881 	 *
1882 	 * Order adding us to wait queue and allocating driver tag.
1883 	 *
1884 	 * The pair is the one implied in sbitmap_queue_wake_up() which
1885 	 * orders clearing sbitmap tag bits and waitqueue_active() in
1886 	 * __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
1887 	 *
1888 	 * Otherwise, re-order of adding wait queue and getting driver tag
1889 	 * may cause __sbitmap_queue_wake_up() to wake up nothing because
1890 	 * the waitqueue_active() may not observe us in wait queue.
1891 	 */
1892 	smp_mb();
1893 
1894 	/*
1895 	 * It's possible that a tag was freed in the window between the
1896 	 * allocation failure and adding the hardware queue to the wait
1897 	 * queue.
1898 	 */
1899 	ret = blk_mq_get_driver_tag(rq);
1900 	if (!ret) {
1901 		spin_unlock(&hctx->dispatch_wait_lock);
1902 		spin_unlock_irq(&wq->lock);
1903 		return false;
1904 	}
1905 
1906 	/*
1907 	 * We got a tag, remove ourselves from the wait queue to ensure
1908 	 * someone else gets the wakeup.
1909 	 */
1910 	list_del_init(&wait->entry);
1911 	atomic_dec(&sbq->ws_active);
1912 	spin_unlock(&hctx->dispatch_wait_lock);
1913 	spin_unlock_irq(&wq->lock);
1914 
1915 	return true;
1916 }
1917 
1918 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
1919 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
1920 /*
1921  * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1922  * - EWMA is one simple way to compute running average value
1923  * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1924  * - take 4 as factor for avoiding to get too small(0) result, and this
1925  *   factor doesn't matter because EWMA decreases exponentially
1926  */
blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx * hctx,bool busy)1927 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1928 {
1929 	unsigned int ewma;
1930 
1931 	ewma = hctx->dispatch_busy;
1932 
1933 	if (!ewma && !busy)
1934 		return;
1935 
1936 	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
1937 	if (busy)
1938 		ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
1939 	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
1940 
1941 	hctx->dispatch_busy = ewma;
1942 }
1943 
1944 #define BLK_MQ_RESOURCE_DELAY	3		/* ms units */
1945 
blk_mq_handle_dev_resource(struct request * rq,struct list_head * list)1946 static void blk_mq_handle_dev_resource(struct request *rq,
1947 				       struct list_head *list)
1948 {
1949 	list_add(&rq->queuelist, list);
1950 	__blk_mq_requeue_request(rq);
1951 }
1952 
blk_mq_handle_zone_resource(struct request * rq,struct list_head * zone_list)1953 static void blk_mq_handle_zone_resource(struct request *rq,
1954 					struct list_head *zone_list)
1955 {
1956 	/*
1957 	 * If we end up here it is because we cannot dispatch a request to a
1958 	 * specific zone due to LLD level zone-write locking or other zone
1959 	 * related resource not being available. In this case, set the request
1960 	 * aside in zone_list for retrying it later.
1961 	 */
1962 	list_add(&rq->queuelist, zone_list);
1963 	__blk_mq_requeue_request(rq);
1964 }
1965 
1966 enum prep_dispatch {
1967 	PREP_DISPATCH_OK,
1968 	PREP_DISPATCH_NO_TAG,
1969 	PREP_DISPATCH_NO_BUDGET,
1970 };
1971 
blk_mq_prep_dispatch_rq(struct request * rq,bool need_budget)1972 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
1973 						  bool need_budget)
1974 {
1975 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1976 	int budget_token = -1;
1977 
1978 	if (need_budget) {
1979 		budget_token = blk_mq_get_dispatch_budget(rq->q);
1980 		if (budget_token < 0) {
1981 			blk_mq_put_driver_tag(rq);
1982 			return PREP_DISPATCH_NO_BUDGET;
1983 		}
1984 		blk_mq_set_rq_budget_token(rq, budget_token);
1985 	}
1986 
1987 	if (!blk_mq_get_driver_tag(rq)) {
1988 		/*
1989 		 * The initial allocation attempt failed, so we need to
1990 		 * rerun the hardware queue when a tag is freed. The
1991 		 * waitqueue takes care of that. If the queue is run
1992 		 * before we add this entry back on the dispatch list,
1993 		 * we'll re-run it below.
1994 		 */
1995 		if (!blk_mq_mark_tag_wait(hctx, rq)) {
1996 			/*
1997 			 * All budgets not got from this function will be put
1998 			 * together during handling partial dispatch
1999 			 */
2000 			if (need_budget)
2001 				blk_mq_put_dispatch_budget(rq->q, budget_token);
2002 			return PREP_DISPATCH_NO_TAG;
2003 		}
2004 	}
2005 
2006 	return PREP_DISPATCH_OK;
2007 }
2008 
2009 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
blk_mq_release_budgets(struct request_queue * q,struct list_head * list)2010 static void blk_mq_release_budgets(struct request_queue *q,
2011 		struct list_head *list)
2012 {
2013 	struct request *rq;
2014 
2015 	list_for_each_entry(rq, list, queuelist) {
2016 		int budget_token = blk_mq_get_rq_budget_token(rq);
2017 
2018 		if (budget_token >= 0)
2019 			blk_mq_put_dispatch_budget(q, budget_token);
2020 	}
2021 }
2022 
2023 /*
2024  * blk_mq_commit_rqs will notify driver using bd->last that there is no
2025  * more requests. (See comment in struct blk_mq_ops for commit_rqs for
2026  * details)
2027  * Attention, we should explicitly call this in unusual cases:
2028  *  1) did not queue everything initially scheduled to queue
2029  *  2) the last attempt to queue a request failed
2030  */
blk_mq_commit_rqs(struct blk_mq_hw_ctx * hctx,int queued,bool from_schedule)2031 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
2032 			      bool from_schedule)
2033 {
2034 	if (hctx->queue->mq_ops->commit_rqs && queued) {
2035 		trace_block_unplug(hctx->queue, queued, !from_schedule);
2036 		hctx->queue->mq_ops->commit_rqs(hctx);
2037 	}
2038 }
2039 
2040 /*
2041  * Returns true if we did some work AND can potentially do more.
2042  */
blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx * hctx,struct list_head * list,unsigned int nr_budgets)2043 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
2044 			     unsigned int nr_budgets)
2045 {
2046 	enum prep_dispatch prep;
2047 	struct request_queue *q = hctx->queue;
2048 	struct request *rq;
2049 	int queued;
2050 	blk_status_t ret = BLK_STS_OK;
2051 	LIST_HEAD(zone_list);
2052 	bool needs_resource = false;
2053 
2054 	if (list_empty(list))
2055 		return false;
2056 
2057 	/*
2058 	 * Now process all the entries, sending them to the driver.
2059 	 */
2060 	queued = 0;
2061 	do {
2062 		struct blk_mq_queue_data bd;
2063 
2064 		rq = list_first_entry(list, struct request, queuelist);
2065 
2066 		WARN_ON_ONCE(hctx != rq->mq_hctx);
2067 		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
2068 		if (prep != PREP_DISPATCH_OK)
2069 			break;
2070 
2071 		list_del_init(&rq->queuelist);
2072 
2073 		bd.rq = rq;
2074 		bd.last = list_empty(list);
2075 
2076 		/*
2077 		 * once the request is queued to lld, no need to cover the
2078 		 * budget any more
2079 		 */
2080 		if (nr_budgets)
2081 			nr_budgets--;
2082 		ret = q->mq_ops->queue_rq(hctx, &bd);
2083 		switch (ret) {
2084 		case BLK_STS_OK:
2085 			queued++;
2086 			break;
2087 		case BLK_STS_RESOURCE:
2088 			needs_resource = true;
2089 			fallthrough;
2090 		case BLK_STS_DEV_RESOURCE:
2091 			blk_mq_handle_dev_resource(rq, list);
2092 			goto out;
2093 		case BLK_STS_ZONE_RESOURCE:
2094 			/*
2095 			 * Move the request to zone_list and keep going through
2096 			 * the dispatch list to find more requests the drive can
2097 			 * accept.
2098 			 */
2099 			blk_mq_handle_zone_resource(rq, &zone_list);
2100 			needs_resource = true;
2101 			break;
2102 		default:
2103 			blk_mq_end_request(rq, ret);
2104 		}
2105 	} while (!list_empty(list));
2106 out:
2107 	if (!list_empty(&zone_list))
2108 		list_splice_tail_init(&zone_list, list);
2109 
2110 	/* If we didn't flush the entire list, we could have told the driver
2111 	 * there was more coming, but that turned out to be a lie.
2112 	 */
2113 	if (!list_empty(list) || ret != BLK_STS_OK)
2114 		blk_mq_commit_rqs(hctx, queued, false);
2115 
2116 	/*
2117 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
2118 	 * that is where we will continue on next queue run.
2119 	 */
2120 	if (!list_empty(list)) {
2121 		bool needs_restart;
2122 		/* For non-shared tags, the RESTART check will suffice */
2123 		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
2124 			((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
2125 			blk_mq_is_shared_tags(hctx->flags));
2126 
2127 		if (nr_budgets)
2128 			blk_mq_release_budgets(q, list);
2129 
2130 		spin_lock(&hctx->lock);
2131 		list_splice_tail_init(list, &hctx->dispatch);
2132 		spin_unlock(&hctx->lock);
2133 
2134 		/*
2135 		 * Order adding requests to hctx->dispatch and checking
2136 		 * SCHED_RESTART flag. The pair of this smp_mb() is the one
2137 		 * in blk_mq_sched_restart(). Avoid restart code path to
2138 		 * miss the new added requests to hctx->dispatch, meantime
2139 		 * SCHED_RESTART is observed here.
2140 		 */
2141 		smp_mb();
2142 
2143 		/*
2144 		 * If SCHED_RESTART was set by the caller of this function and
2145 		 * it is no longer set that means that it was cleared by another
2146 		 * thread and hence that a queue rerun is needed.
2147 		 *
2148 		 * If 'no_tag' is set, that means that we failed getting
2149 		 * a driver tag with an I/O scheduler attached. If our dispatch
2150 		 * waitqueue is no longer active, ensure that we run the queue
2151 		 * AFTER adding our entries back to the list.
2152 		 *
2153 		 * If no I/O scheduler has been configured it is possible that
2154 		 * the hardware queue got stopped and restarted before requests
2155 		 * were pushed back onto the dispatch list. Rerun the queue to
2156 		 * avoid starvation. Notes:
2157 		 * - blk_mq_run_hw_queue() checks whether or not a queue has
2158 		 *   been stopped before rerunning a queue.
2159 		 * - Some but not all block drivers stop a queue before
2160 		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
2161 		 *   and dm-rq.
2162 		 *
2163 		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
2164 		 * bit is set, run queue after a delay to avoid IO stalls
2165 		 * that could otherwise occur if the queue is idle.  We'll do
2166 		 * similar if we couldn't get budget or couldn't lock a zone
2167 		 * and SCHED_RESTART is set.
2168 		 */
2169 		needs_restart = blk_mq_sched_needs_restart(hctx);
2170 		if (prep == PREP_DISPATCH_NO_BUDGET)
2171 			needs_resource = true;
2172 		if (!needs_restart ||
2173 		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
2174 			blk_mq_run_hw_queue(hctx, true);
2175 		else if (needs_resource)
2176 			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
2177 
2178 		blk_mq_update_dispatch_busy(hctx, true);
2179 		return false;
2180 	}
2181 
2182 	blk_mq_update_dispatch_busy(hctx, false);
2183 	return true;
2184 }
2185 
blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx * hctx)2186 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
2187 {
2188 	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
2189 
2190 	if (cpu >= nr_cpu_ids)
2191 		cpu = cpumask_first(hctx->cpumask);
2192 	return cpu;
2193 }
2194 
2195 /*
2196  * It'd be great if the workqueue API had a way to pass
2197  * in a mask and had some smarts for more clever placement.
2198  * For now we just round-robin here, switching for every
2199  * BLK_MQ_CPU_WORK_BATCH queued items.
2200  */
blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx * hctx)2201 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
2202 {
2203 	bool tried = false;
2204 	int next_cpu = hctx->next_cpu;
2205 
2206 	if (hctx->queue->nr_hw_queues == 1)
2207 		return WORK_CPU_UNBOUND;
2208 
2209 	if (--hctx->next_cpu_batch <= 0) {
2210 select_cpu:
2211 		next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
2212 				cpu_online_mask);
2213 		if (next_cpu >= nr_cpu_ids)
2214 			next_cpu = blk_mq_first_mapped_cpu(hctx);
2215 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2216 	}
2217 
2218 	/*
2219 	 * Do unbound schedule if we can't find a online CPU for this hctx,
2220 	 * and it should only happen in the path of handling CPU DEAD.
2221 	 */
2222 	if (!cpu_online(next_cpu)) {
2223 		if (!tried) {
2224 			tried = true;
2225 			goto select_cpu;
2226 		}
2227 
2228 		/*
2229 		 * Make sure to re-select CPU next time once after CPUs
2230 		 * in hctx->cpumask become online again.
2231 		 */
2232 		hctx->next_cpu = next_cpu;
2233 		hctx->next_cpu_batch = 1;
2234 		return WORK_CPU_UNBOUND;
2235 	}
2236 
2237 	hctx->next_cpu = next_cpu;
2238 	return next_cpu;
2239 }
2240 
2241 /**
2242  * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
2243  * @hctx: Pointer to the hardware queue to run.
2244  * @msecs: Milliseconds of delay to wait before running the queue.
2245  *
2246  * Run a hardware queue asynchronously with a delay of @msecs.
2247  */
blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx * hctx,unsigned long msecs)2248 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
2249 {
2250 	if (unlikely(blk_mq_hctx_stopped(hctx)))
2251 		return;
2252 	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
2253 				    msecs_to_jiffies(msecs));
2254 }
2255 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
2256 
2257 /**
2258  * blk_mq_run_hw_queue - Start to run a hardware queue.
2259  * @hctx: Pointer to the hardware queue to run.
2260  * @async: If we want to run the queue asynchronously.
2261  *
2262  * Check if the request queue is not in a quiesced state and if there are
2263  * pending requests to be sent. If this is true, run the queue to send requests
2264  * to hardware.
2265  */
blk_mq_run_hw_queue(struct blk_mq_hw_ctx * hctx,bool async)2266 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2267 {
2268 	bool need_run;
2269 
2270 	/*
2271 	 * We can't run the queue inline with interrupts disabled.
2272 	 */
2273 	WARN_ON_ONCE(!async && in_interrupt());
2274 
2275 	might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
2276 
2277 	/*
2278 	 * When queue is quiesced, we may be switching io scheduler, or
2279 	 * updating nr_hw_queues, or other things, and we can't run queue
2280 	 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
2281 	 *
2282 	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
2283 	 * quiesced.
2284 	 */
2285 	__blk_mq_run_dispatch_ops(hctx->queue, false,
2286 		need_run = !blk_queue_quiesced(hctx->queue) &&
2287 		blk_mq_hctx_has_pending(hctx));
2288 
2289 	if (!need_run)
2290 		return;
2291 
2292 	if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
2293 		blk_mq_delay_run_hw_queue(hctx, 0);
2294 		return;
2295 	}
2296 
2297 	blk_mq_run_dispatch_ops(hctx->queue,
2298 				blk_mq_sched_dispatch_requests(hctx));
2299 }
2300 EXPORT_SYMBOL(blk_mq_run_hw_queue);
2301 
2302 /*
2303  * Return prefered queue to dispatch from (if any) for non-mq aware IO
2304  * scheduler.
2305  */
blk_mq_get_sq_hctx(struct request_queue * q)2306 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
2307 {
2308 	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
2309 	/*
2310 	 * If the IO scheduler does not respect hardware queues when
2311 	 * dispatching, we just don't bother with multiple HW queues and
2312 	 * dispatch from hctx for the current CPU since running multiple queues
2313 	 * just causes lock contention inside the scheduler and pointless cache
2314 	 * bouncing.
2315 	 */
2316 	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
2317 
2318 	if (!blk_mq_hctx_stopped(hctx))
2319 		return hctx;
2320 	return NULL;
2321 }
2322 
2323 /**
2324  * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
2325  * @q: Pointer to the request queue to run.
2326  * @async: If we want to run the queue asynchronously.
2327  */
blk_mq_run_hw_queues(struct request_queue * q,bool async)2328 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
2329 {
2330 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
2331 	unsigned long i;
2332 
2333 	sq_hctx = NULL;
2334 	if (blk_queue_sq_sched(q))
2335 		sq_hctx = blk_mq_get_sq_hctx(q);
2336 	queue_for_each_hw_ctx(q, hctx, i) {
2337 		if (blk_mq_hctx_stopped(hctx))
2338 			continue;
2339 		/*
2340 		 * Dispatch from this hctx either if there's no hctx preferred
2341 		 * by IO scheduler or if it has requests that bypass the
2342 		 * scheduler.
2343 		 */
2344 		if (!sq_hctx || sq_hctx == hctx ||
2345 		    !list_empty_careful(&hctx->dispatch))
2346 			blk_mq_run_hw_queue(hctx, async);
2347 	}
2348 }
2349 EXPORT_SYMBOL(blk_mq_run_hw_queues);
2350 
2351 /**
2352  * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
2353  * @q: Pointer to the request queue to run.
2354  * @msecs: Milliseconds of delay to wait before running the queues.
2355  */
blk_mq_delay_run_hw_queues(struct request_queue * q,unsigned long msecs)2356 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
2357 {
2358 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
2359 	unsigned long i;
2360 
2361 	sq_hctx = NULL;
2362 	if (blk_queue_sq_sched(q))
2363 		sq_hctx = blk_mq_get_sq_hctx(q);
2364 	queue_for_each_hw_ctx(q, hctx, i) {
2365 		if (blk_mq_hctx_stopped(hctx))
2366 			continue;
2367 		/*
2368 		 * If there is already a run_work pending, leave the
2369 		 * pending delay untouched. Otherwise, a hctx can stall
2370 		 * if another hctx is re-delaying the other's work
2371 		 * before the work executes.
2372 		 */
2373 		if (delayed_work_pending(&hctx->run_work))
2374 			continue;
2375 		/*
2376 		 * Dispatch from this hctx either if there's no hctx preferred
2377 		 * by IO scheduler or if it has requests that bypass the
2378 		 * scheduler.
2379 		 */
2380 		if (!sq_hctx || sq_hctx == hctx ||
2381 		    !list_empty_careful(&hctx->dispatch))
2382 			blk_mq_delay_run_hw_queue(hctx, msecs);
2383 	}
2384 }
2385 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
2386 
2387 /*
2388  * This function is often used for pausing .queue_rq() by driver when
2389  * there isn't enough resource or some conditions aren't satisfied, and
2390  * BLK_STS_RESOURCE is usually returned.
2391  *
2392  * We do not guarantee that dispatch can be drained or blocked
2393  * after blk_mq_stop_hw_queue() returns. Please use
2394  * blk_mq_quiesce_queue() for that requirement.
2395  */
blk_mq_stop_hw_queue(struct blk_mq_hw_ctx * hctx)2396 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
2397 {
2398 	cancel_delayed_work(&hctx->run_work);
2399 
2400 	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2401 }
2402 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2403 
2404 /*
2405  * This function is often used for pausing .queue_rq() by driver when
2406  * there isn't enough resource or some conditions aren't satisfied, and
2407  * BLK_STS_RESOURCE is usually returned.
2408  *
2409  * We do not guarantee that dispatch can be drained or blocked
2410  * after blk_mq_stop_hw_queues() returns. Please use
2411  * blk_mq_quiesce_queue() for that requirement.
2412  */
blk_mq_stop_hw_queues(struct request_queue * q)2413 void blk_mq_stop_hw_queues(struct request_queue *q)
2414 {
2415 	struct blk_mq_hw_ctx *hctx;
2416 	unsigned long i;
2417 
2418 	queue_for_each_hw_ctx(q, hctx, i)
2419 		blk_mq_stop_hw_queue(hctx);
2420 }
2421 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
2422 
blk_mq_start_hw_queue(struct blk_mq_hw_ctx * hctx)2423 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
2424 {
2425 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2426 
2427 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
2428 }
2429 EXPORT_SYMBOL(blk_mq_start_hw_queue);
2430 
blk_mq_start_hw_queues(struct request_queue * q)2431 void blk_mq_start_hw_queues(struct request_queue *q)
2432 {
2433 	struct blk_mq_hw_ctx *hctx;
2434 	unsigned long i;
2435 
2436 	queue_for_each_hw_ctx(q, hctx, i)
2437 		blk_mq_start_hw_queue(hctx);
2438 }
2439 EXPORT_SYMBOL(blk_mq_start_hw_queues);
2440 
blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx * hctx,bool async)2441 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2442 {
2443 	if (!blk_mq_hctx_stopped(hctx))
2444 		return;
2445 
2446 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2447 	blk_mq_run_hw_queue(hctx, async);
2448 }
2449 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
2450 
blk_mq_start_stopped_hw_queues(struct request_queue * q,bool async)2451 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
2452 {
2453 	struct blk_mq_hw_ctx *hctx;
2454 	unsigned long i;
2455 
2456 	queue_for_each_hw_ctx(q, hctx, i)
2457 		blk_mq_start_stopped_hw_queue(hctx, async ||
2458 					(hctx->flags & BLK_MQ_F_BLOCKING));
2459 }
2460 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
2461 
blk_mq_run_work_fn(struct work_struct * work)2462 static void blk_mq_run_work_fn(struct work_struct *work)
2463 {
2464 	struct blk_mq_hw_ctx *hctx =
2465 		container_of(work, struct blk_mq_hw_ctx, run_work.work);
2466 
2467 	blk_mq_run_dispatch_ops(hctx->queue,
2468 				blk_mq_sched_dispatch_requests(hctx));
2469 }
2470 
2471 /**
2472  * blk_mq_request_bypass_insert - Insert a request at dispatch list.
2473  * @rq: Pointer to request to be inserted.
2474  * @flags: BLK_MQ_INSERT_*
2475  *
2476  * Should only be used carefully, when the caller knows we want to
2477  * bypass a potential IO scheduler on the target device.
2478  */
blk_mq_request_bypass_insert(struct request * rq,blk_insert_t flags)2479 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
2480 {
2481 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2482 
2483 	spin_lock(&hctx->lock);
2484 	if (flags & BLK_MQ_INSERT_AT_HEAD)
2485 		list_add(&rq->queuelist, &hctx->dispatch);
2486 	else
2487 		list_add_tail(&rq->queuelist, &hctx->dispatch);
2488 	spin_unlock(&hctx->lock);
2489 }
2490 
blk_mq_insert_requests(struct blk_mq_hw_ctx * hctx,struct blk_mq_ctx * ctx,struct list_head * list,bool run_queue_async)2491 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
2492 		struct blk_mq_ctx *ctx, struct list_head *list,
2493 		bool run_queue_async)
2494 {
2495 	struct request *rq;
2496 	enum hctx_type type = hctx->type;
2497 
2498 	/*
2499 	 * Try to issue requests directly if the hw queue isn't busy to save an
2500 	 * extra enqueue & dequeue to the sw queue.
2501 	 */
2502 	if (!hctx->dispatch_busy && !run_queue_async) {
2503 		blk_mq_run_dispatch_ops(hctx->queue,
2504 			blk_mq_try_issue_list_directly(hctx, list));
2505 		if (list_empty(list))
2506 			goto out;
2507 	}
2508 
2509 	/*
2510 	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
2511 	 * offline now
2512 	 */
2513 	list_for_each_entry(rq, list, queuelist) {
2514 		BUG_ON(rq->mq_ctx != ctx);
2515 		trace_block_rq_insert(rq);
2516 		if (rq->cmd_flags & REQ_NOWAIT)
2517 			run_queue_async = true;
2518 	}
2519 
2520 	spin_lock(&ctx->lock);
2521 	list_splice_tail_init(list, &ctx->rq_lists[type]);
2522 	blk_mq_hctx_mark_pending(hctx, ctx);
2523 	spin_unlock(&ctx->lock);
2524 out:
2525 	blk_mq_run_hw_queue(hctx, run_queue_async);
2526 }
2527 
blk_mq_insert_request(struct request * rq,blk_insert_t flags)2528 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
2529 {
2530 	struct request_queue *q = rq->q;
2531 	struct blk_mq_ctx *ctx = rq->mq_ctx;
2532 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2533 
2534 	if (blk_rq_is_passthrough(rq)) {
2535 		/*
2536 		 * Passthrough request have to be added to hctx->dispatch
2537 		 * directly.  The device may be in a situation where it can't
2538 		 * handle FS request, and always returns BLK_STS_RESOURCE for
2539 		 * them, which gets them added to hctx->dispatch.
2540 		 *
2541 		 * If a passthrough request is required to unblock the queues,
2542 		 * and it is added to the scheduler queue, there is no chance to
2543 		 * dispatch it given we prioritize requests in hctx->dispatch.
2544 		 */
2545 		blk_mq_request_bypass_insert(rq, flags);
2546 	} else if (req_op(rq) == REQ_OP_FLUSH) {
2547 		/*
2548 		 * Firstly normal IO request is inserted to scheduler queue or
2549 		 * sw queue, meantime we add flush request to dispatch queue(
2550 		 * hctx->dispatch) directly and there is at most one in-flight
2551 		 * flush request for each hw queue, so it doesn't matter to add
2552 		 * flush request to tail or front of the dispatch queue.
2553 		 *
2554 		 * Secondly in case of NCQ, flush request belongs to non-NCQ
2555 		 * command, and queueing it will fail when there is any
2556 		 * in-flight normal IO request(NCQ command). When adding flush
2557 		 * rq to the front of hctx->dispatch, it is easier to introduce
2558 		 * extra time to flush rq's latency because of S_SCHED_RESTART
2559 		 * compared with adding to the tail of dispatch queue, then
2560 		 * chance of flush merge is increased, and less flush requests
2561 		 * will be issued to controller. It is observed that ~10% time
2562 		 * is saved in blktests block/004 on disk attached to AHCI/NCQ
2563 		 * drive when adding flush rq to the front of hctx->dispatch.
2564 		 *
2565 		 * Simply queue flush rq to the front of hctx->dispatch so that
2566 		 * intensive flush workloads can benefit in case of NCQ HW.
2567 		 */
2568 		blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
2569 	} else if (q->elevator) {
2570 		LIST_HEAD(list);
2571 
2572 		WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
2573 
2574 		list_add(&rq->queuelist, &list);
2575 		q->elevator->type->ops.insert_requests(hctx, &list, flags);
2576 	} else {
2577 		trace_block_rq_insert(rq);
2578 
2579 		spin_lock(&ctx->lock);
2580 		if (flags & BLK_MQ_INSERT_AT_HEAD)
2581 			list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
2582 		else
2583 			list_add_tail(&rq->queuelist,
2584 				      &ctx->rq_lists[hctx->type]);
2585 		blk_mq_hctx_mark_pending(hctx, ctx);
2586 		spin_unlock(&ctx->lock);
2587 	}
2588 }
2589 
blk_mq_bio_to_request(struct request * rq,struct bio * bio,unsigned int nr_segs)2590 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
2591 		unsigned int nr_segs)
2592 {
2593 	int err;
2594 
2595 	if (bio->bi_opf & REQ_RAHEAD)
2596 		rq->cmd_flags |= REQ_FAILFAST_MASK;
2597 
2598 	rq->__sector = bio->bi_iter.bi_sector;
2599 	blk_rq_bio_prep(rq, bio, nr_segs);
2600 
2601 	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2602 	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2603 	WARN_ON_ONCE(err);
2604 
2605 	blk_account_io_start(rq);
2606 }
2607 
__blk_mq_issue_directly(struct blk_mq_hw_ctx * hctx,struct request * rq,bool last)2608 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
2609 					    struct request *rq, bool last)
2610 {
2611 	struct request_queue *q = rq->q;
2612 	struct blk_mq_queue_data bd = {
2613 		.rq = rq,
2614 		.last = last,
2615 	};
2616 	blk_status_t ret;
2617 
2618 	/*
2619 	 * For OK queue, we are done. For error, caller may kill it.
2620 	 * Any other error (busy), just add it to our list as we
2621 	 * previously would have done.
2622 	 */
2623 	ret = q->mq_ops->queue_rq(hctx, &bd);
2624 	switch (ret) {
2625 	case BLK_STS_OK:
2626 		blk_mq_update_dispatch_busy(hctx, false);
2627 		break;
2628 	case BLK_STS_RESOURCE:
2629 	case BLK_STS_DEV_RESOURCE:
2630 		blk_mq_update_dispatch_busy(hctx, true);
2631 		__blk_mq_requeue_request(rq);
2632 		break;
2633 	default:
2634 		blk_mq_update_dispatch_busy(hctx, false);
2635 		break;
2636 	}
2637 
2638 	return ret;
2639 }
2640 
blk_mq_get_budget_and_tag(struct request * rq)2641 static bool blk_mq_get_budget_and_tag(struct request *rq)
2642 {
2643 	int budget_token;
2644 
2645 	budget_token = blk_mq_get_dispatch_budget(rq->q);
2646 	if (budget_token < 0)
2647 		return false;
2648 	blk_mq_set_rq_budget_token(rq, budget_token);
2649 	if (!blk_mq_get_driver_tag(rq)) {
2650 		blk_mq_put_dispatch_budget(rq->q, budget_token);
2651 		return false;
2652 	}
2653 	return true;
2654 }
2655 
2656 /**
2657  * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2658  * @hctx: Pointer of the associated hardware queue.
2659  * @rq: Pointer to request to be sent.
2660  *
2661  * If the device has enough resources to accept a new request now, send the
2662  * request directly to device driver. Else, insert at hctx->dispatch queue, so
2663  * we can try send it another time in the future. Requests inserted at this
2664  * queue have higher priority.
2665  */
blk_mq_try_issue_directly(struct blk_mq_hw_ctx * hctx,struct request * rq)2666 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2667 		struct request *rq)
2668 {
2669 	blk_status_t ret;
2670 
2671 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2672 		blk_mq_insert_request(rq, 0);
2673 		return;
2674 	}
2675 
2676 	if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
2677 		blk_mq_insert_request(rq, 0);
2678 		blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
2679 		return;
2680 	}
2681 
2682 	ret = __blk_mq_issue_directly(hctx, rq, true);
2683 	switch (ret) {
2684 	case BLK_STS_OK:
2685 		break;
2686 	case BLK_STS_RESOURCE:
2687 	case BLK_STS_DEV_RESOURCE:
2688 		blk_mq_request_bypass_insert(rq, 0);
2689 		blk_mq_run_hw_queue(hctx, false);
2690 		break;
2691 	default:
2692 		blk_mq_end_request(rq, ret);
2693 		break;
2694 	}
2695 }
2696 
blk_mq_request_issue_directly(struct request * rq,bool last)2697 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2698 {
2699 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2700 
2701 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2702 		blk_mq_insert_request(rq, 0);
2703 		return BLK_STS_OK;
2704 	}
2705 
2706 	if (!blk_mq_get_budget_and_tag(rq))
2707 		return BLK_STS_RESOURCE;
2708 	return __blk_mq_issue_directly(hctx, rq, last);
2709 }
2710 
blk_mq_plug_issue_direct(struct blk_plug * plug)2711 static void blk_mq_plug_issue_direct(struct blk_plug *plug)
2712 {
2713 	struct blk_mq_hw_ctx *hctx = NULL;
2714 	struct request *rq;
2715 	int queued = 0;
2716 	blk_status_t ret = BLK_STS_OK;
2717 
2718 	while ((rq = rq_list_pop(&plug->mq_list))) {
2719 		bool last = rq_list_empty(plug->mq_list);
2720 
2721 		if (hctx != rq->mq_hctx) {
2722 			if (hctx) {
2723 				blk_mq_commit_rqs(hctx, queued, false);
2724 				queued = 0;
2725 			}
2726 			hctx = rq->mq_hctx;
2727 		}
2728 
2729 		ret = blk_mq_request_issue_directly(rq, last);
2730 		switch (ret) {
2731 		case BLK_STS_OK:
2732 			queued++;
2733 			break;
2734 		case BLK_STS_RESOURCE:
2735 		case BLK_STS_DEV_RESOURCE:
2736 			blk_mq_request_bypass_insert(rq, 0);
2737 			blk_mq_run_hw_queue(hctx, false);
2738 			goto out;
2739 		default:
2740 			blk_mq_end_request(rq, ret);
2741 			break;
2742 		}
2743 	}
2744 
2745 out:
2746 	if (ret != BLK_STS_OK)
2747 		blk_mq_commit_rqs(hctx, queued, false);
2748 }
2749 
__blk_mq_flush_plug_list(struct request_queue * q,struct blk_plug * plug)2750 static void __blk_mq_flush_plug_list(struct request_queue *q,
2751 				     struct blk_plug *plug)
2752 {
2753 	if (blk_queue_quiesced(q))
2754 		return;
2755 	q->mq_ops->queue_rqs(&plug->mq_list);
2756 }
2757 
blk_mq_dispatch_plug_list(struct blk_plug * plug,bool from_sched)2758 static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)
2759 {
2760 	struct blk_mq_hw_ctx *this_hctx = NULL;
2761 	struct blk_mq_ctx *this_ctx = NULL;
2762 	struct request *requeue_list = NULL;
2763 	struct request **requeue_lastp = &requeue_list;
2764 	unsigned int depth = 0;
2765 	bool is_passthrough = false;
2766 	LIST_HEAD(list);
2767 
2768 	do {
2769 		struct request *rq = rq_list_pop(&plug->mq_list);
2770 
2771 		if (!this_hctx) {
2772 			this_hctx = rq->mq_hctx;
2773 			this_ctx = rq->mq_ctx;
2774 			is_passthrough = blk_rq_is_passthrough(rq);
2775 		} else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
2776 			   is_passthrough != blk_rq_is_passthrough(rq)) {
2777 			rq_list_add_tail(&requeue_lastp, rq);
2778 			continue;
2779 		}
2780 		list_add(&rq->queuelist, &list);
2781 		depth++;
2782 	} while (!rq_list_empty(plug->mq_list));
2783 
2784 	plug->mq_list = requeue_list;
2785 	trace_block_unplug(this_hctx->queue, depth, !from_sched);
2786 
2787 	percpu_ref_get(&this_hctx->queue->q_usage_counter);
2788 	/* passthrough requests should never be issued to the I/O scheduler */
2789 	if (is_passthrough) {
2790 		spin_lock(&this_hctx->lock);
2791 		list_splice_tail_init(&list, &this_hctx->dispatch);
2792 		spin_unlock(&this_hctx->lock);
2793 		blk_mq_run_hw_queue(this_hctx, from_sched);
2794 	} else if (this_hctx->queue->elevator) {
2795 		this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
2796 				&list, 0);
2797 		blk_mq_run_hw_queue(this_hctx, from_sched);
2798 	} else {
2799 		blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
2800 	}
2801 	percpu_ref_put(&this_hctx->queue->q_usage_counter);
2802 }
2803 
blk_mq_flush_plug_list(struct blk_plug * plug,bool from_schedule)2804 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2805 {
2806 	struct request *rq;
2807 
2808 	/*
2809 	 * We may have been called recursively midway through handling
2810 	 * plug->mq_list via a schedule() in the driver's queue_rq() callback.
2811 	 * To avoid mq_list changing under our feet, clear rq_count early and
2812 	 * bail out specifically if rq_count is 0 rather than checking
2813 	 * whether the mq_list is empty.
2814 	 */
2815 	if (plug->rq_count == 0)
2816 		return;
2817 	plug->rq_count = 0;
2818 
2819 	if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) {
2820 		struct request_queue *q;
2821 
2822 		rq = rq_list_peek(&plug->mq_list);
2823 		q = rq->q;
2824 
2825 		/*
2826 		 * Peek first request and see if we have a ->queue_rqs() hook.
2827 		 * If we do, we can dispatch the whole plug list in one go. We
2828 		 * already know at this point that all requests belong to the
2829 		 * same queue, caller must ensure that's the case.
2830 		 *
2831 		 * Since we pass off the full list to the driver at this point,
2832 		 * we do not increment the active request count for the queue.
2833 		 * Bypass shared tags for now because of that.
2834 		 */
2835 		if (q->mq_ops->queue_rqs &&
2836 		    !(rq->mq_hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
2837 			blk_mq_run_dispatch_ops(q,
2838 				__blk_mq_flush_plug_list(q, plug));
2839 			if (rq_list_empty(plug->mq_list))
2840 				return;
2841 		}
2842 
2843 		blk_mq_run_dispatch_ops(q,
2844 				blk_mq_plug_issue_direct(plug));
2845 		if (rq_list_empty(plug->mq_list))
2846 			return;
2847 	}
2848 
2849 	do {
2850 		blk_mq_dispatch_plug_list(plug, from_schedule);
2851 	} while (!rq_list_empty(plug->mq_list));
2852 }
2853 
blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx * hctx,struct list_head * list)2854 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2855 		struct list_head *list)
2856 {
2857 	int queued = 0;
2858 	blk_status_t ret = BLK_STS_OK;
2859 
2860 	while (!list_empty(list)) {
2861 		struct request *rq = list_first_entry(list, struct request,
2862 				queuelist);
2863 
2864 		list_del_init(&rq->queuelist);
2865 		ret = blk_mq_request_issue_directly(rq, list_empty(list));
2866 		switch (ret) {
2867 		case BLK_STS_OK:
2868 			queued++;
2869 			break;
2870 		case BLK_STS_RESOURCE:
2871 		case BLK_STS_DEV_RESOURCE:
2872 			blk_mq_request_bypass_insert(rq, 0);
2873 			if (list_empty(list))
2874 				blk_mq_run_hw_queue(hctx, false);
2875 			goto out;
2876 		default:
2877 			blk_mq_end_request(rq, ret);
2878 			break;
2879 		}
2880 	}
2881 
2882 out:
2883 	if (ret != BLK_STS_OK)
2884 		blk_mq_commit_rqs(hctx, queued, false);
2885 }
2886 
blk_mq_attempt_bio_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs)2887 static bool blk_mq_attempt_bio_merge(struct request_queue *q,
2888 				     struct bio *bio, unsigned int nr_segs)
2889 {
2890 	if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
2891 		if (blk_attempt_plug_merge(q, bio, nr_segs))
2892 			return true;
2893 		if (blk_mq_sched_bio_merge(q, bio, nr_segs))
2894 			return true;
2895 	}
2896 	return false;
2897 }
2898 
blk_mq_get_new_requests(struct request_queue * q,struct blk_plug * plug,struct bio * bio,unsigned int nsegs)2899 static struct request *blk_mq_get_new_requests(struct request_queue *q,
2900 					       struct blk_plug *plug,
2901 					       struct bio *bio,
2902 					       unsigned int nsegs)
2903 {
2904 	struct blk_mq_alloc_data data = {
2905 		.q		= q,
2906 		.nr_tags	= 1,
2907 		.cmd_flags	= bio->bi_opf,
2908 	};
2909 	struct request *rq;
2910 
2911 	if (blk_mq_attempt_bio_merge(q, bio, nsegs))
2912 		return NULL;
2913 
2914 	rq_qos_throttle(q, bio);
2915 
2916 	if (plug) {
2917 		data.nr_tags = plug->nr_ios;
2918 		plug->nr_ios = 1;
2919 		data.cached_rq = &plug->cached_rq;
2920 	}
2921 
2922 	rq = __blk_mq_alloc_requests(&data);
2923 	if (rq)
2924 		return rq;
2925 	rq_qos_cleanup(q, bio);
2926 	if (bio->bi_opf & REQ_NOWAIT)
2927 		bio_wouldblock_error(bio);
2928 	return NULL;
2929 }
2930 
2931 /* return true if this @rq can be used for @bio */
blk_mq_can_use_cached_rq(struct request * rq,struct blk_plug * plug,struct bio * bio)2932 static bool blk_mq_can_use_cached_rq(struct request *rq, struct blk_plug *plug,
2933 		struct bio *bio)
2934 {
2935 	enum hctx_type type = blk_mq_get_hctx_type(bio->bi_opf);
2936 	enum hctx_type hctx_type = rq->mq_hctx->type;
2937 
2938 	WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);
2939 
2940 	if (type != hctx_type &&
2941 	    !(type == HCTX_TYPE_READ && hctx_type == HCTX_TYPE_DEFAULT))
2942 		return false;
2943 	if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf))
2944 		return false;
2945 
2946 	/*
2947 	 * If any qos ->throttle() end up blocking, we will have flushed the
2948 	 * plug and hence killed the cached_rq list as well. Pop this entry
2949 	 * before we throttle.
2950 	 */
2951 	plug->cached_rq = rq_list_next(rq);
2952 	rq_qos_throttle(rq->q, bio);
2953 
2954 	blk_mq_rq_time_init(rq, 0);
2955 	rq->cmd_flags = bio->bi_opf;
2956 	INIT_LIST_HEAD(&rq->queuelist);
2957 	return true;
2958 }
2959 
bio_set_ioprio(struct bio * bio)2960 static void bio_set_ioprio(struct bio *bio)
2961 {
2962 	/* Nobody set ioprio so far? Initialize it based on task's nice value */
2963 	if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
2964 		bio->bi_ioprio = get_current_ioprio();
2965 	blkcg_set_ioprio(bio);
2966 }
2967 
2968 /**
2969  * blk_mq_submit_bio - Create and send a request to block device.
2970  * @bio: Bio pointer.
2971  *
2972  * Builds up a request structure from @q and @bio and send to the device. The
2973  * request may not be queued directly to hardware if:
2974  * * This request can be merged with another one
2975  * * We want to place request at plug queue for possible future merging
2976  * * There is an IO scheduler active at this queue
2977  *
2978  * It will not queue the request if there is an error with the bio, or at the
2979  * request creation.
2980  */
blk_mq_submit_bio(struct bio * bio)2981 void blk_mq_submit_bio(struct bio *bio)
2982 {
2983 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
2984 	struct blk_plug *plug = blk_mq_plug(bio);
2985 	const int is_sync = op_is_sync(bio->bi_opf);
2986 	struct blk_mq_hw_ctx *hctx;
2987 	struct request *rq = NULL;
2988 	unsigned int nr_segs = 1;
2989 	blk_status_t ret;
2990 
2991 	bio = blk_queue_bounce(bio, q);
2992 	bio_set_ioprio(bio);
2993 
2994 	if (plug) {
2995 		rq = rq_list_peek(&plug->cached_rq);
2996 		if (rq && rq->q != q)
2997 			rq = NULL;
2998 	}
2999 	if (rq) {
3000 		if (unlikely(bio_may_exceed_limits(bio, &q->limits))) {
3001 			bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3002 			if (!bio)
3003 				return;
3004 		}
3005 		if (!bio_integrity_prep(bio))
3006 			return;
3007 		if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
3008 			return;
3009 		if (blk_mq_can_use_cached_rq(rq, plug, bio))
3010 			goto done;
3011 		percpu_ref_get(&q->q_usage_counter);
3012 	} else {
3013 		if (unlikely(bio_queue_enter(bio)))
3014 			return;
3015 		if (unlikely(bio_may_exceed_limits(bio, &q->limits))) {
3016 			bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3017 			if (!bio)
3018 				goto fail;
3019 		}
3020 		if (!bio_integrity_prep(bio))
3021 			goto fail;
3022 	}
3023 
3024 	rq = blk_mq_get_new_requests(q, plug, bio, nr_segs);
3025 	if (unlikely(!rq)) {
3026 fail:
3027 		blk_queue_exit(q);
3028 		return;
3029 	}
3030 
3031 done:
3032 	trace_block_getrq(bio);
3033 
3034 	rq_qos_track(q, rq, bio);
3035 
3036 	blk_mq_bio_to_request(rq, bio, nr_segs);
3037 
3038 	ret = blk_crypto_rq_get_keyslot(rq);
3039 	if (ret != BLK_STS_OK) {
3040 		bio->bi_status = ret;
3041 		bio_endio(bio);
3042 		blk_mq_free_request(rq);
3043 		return;
3044 	}
3045 
3046 	if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
3047 		return;
3048 
3049 	if (plug) {
3050 		blk_add_rq_to_plug(plug, rq);
3051 		return;
3052 	}
3053 
3054 	hctx = rq->mq_hctx;
3055 	if ((rq->rq_flags & RQF_USE_SCHED) ||
3056 	    (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
3057 		blk_mq_insert_request(rq, 0);
3058 		blk_mq_run_hw_queue(hctx, true);
3059 	} else {
3060 		blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
3061 	}
3062 }
3063 
3064 #ifdef CONFIG_BLK_MQ_STACKING
3065 /**
3066  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
3067  * @rq: the request being queued
3068  */
blk_insert_cloned_request(struct request * rq)3069 blk_status_t blk_insert_cloned_request(struct request *rq)
3070 {
3071 	struct request_queue *q = rq->q;
3072 	unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
3073 	unsigned int max_segments = blk_rq_get_max_segments(rq);
3074 	blk_status_t ret;
3075 
3076 	if (blk_rq_sectors(rq) > max_sectors) {
3077 		/*
3078 		 * SCSI device does not have a good way to return if
3079 		 * Write Same/Zero is actually supported. If a device rejects
3080 		 * a non-read/write command (discard, write same,etc.) the
3081 		 * low-level device driver will set the relevant queue limit to
3082 		 * 0 to prevent blk-lib from issuing more of the offending
3083 		 * operations. Commands queued prior to the queue limit being
3084 		 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
3085 		 * errors being propagated to upper layers.
3086 		 */
3087 		if (max_sectors == 0)
3088 			return BLK_STS_NOTSUPP;
3089 
3090 		printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
3091 			__func__, blk_rq_sectors(rq), max_sectors);
3092 		return BLK_STS_IOERR;
3093 	}
3094 
3095 	/*
3096 	 * The queue settings related to segment counting may differ from the
3097 	 * original queue.
3098 	 */
3099 	rq->nr_phys_segments = blk_recalc_rq_segments(rq);
3100 	if (rq->nr_phys_segments > max_segments) {
3101 		printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
3102 			__func__, rq->nr_phys_segments, max_segments);
3103 		return BLK_STS_IOERR;
3104 	}
3105 
3106 	if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
3107 		return BLK_STS_IOERR;
3108 
3109 	ret = blk_crypto_rq_get_keyslot(rq);
3110 	if (ret != BLK_STS_OK)
3111 		return ret;
3112 
3113 	blk_account_io_start(rq);
3114 
3115 	/*
3116 	 * Since we have a scheduler attached on the top device,
3117 	 * bypass a potential scheduler on the bottom device for
3118 	 * insert.
3119 	 */
3120 	blk_mq_run_dispatch_ops(q,
3121 			ret = blk_mq_request_issue_directly(rq, true));
3122 	if (ret)
3123 		blk_account_io_done(rq, ktime_get_ns());
3124 	return ret;
3125 }
3126 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
3127 
3128 /**
3129  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3130  * @rq: the clone request to be cleaned up
3131  *
3132  * Description:
3133  *     Free all bios in @rq for a cloned request.
3134  */
blk_rq_unprep_clone(struct request * rq)3135 void blk_rq_unprep_clone(struct request *rq)
3136 {
3137 	struct bio *bio;
3138 
3139 	while ((bio = rq->bio) != NULL) {
3140 		rq->bio = bio->bi_next;
3141 
3142 		bio_put(bio);
3143 	}
3144 }
3145 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3146 
3147 /**
3148  * blk_rq_prep_clone - Helper function to setup clone request
3149  * @rq: the request to be setup
3150  * @rq_src: original request to be cloned
3151  * @bs: bio_set that bios for clone are allocated from
3152  * @gfp_mask: memory allocation mask for bio
3153  * @bio_ctr: setup function to be called for each clone bio.
3154  *           Returns %0 for success, non %0 for failure.
3155  * @data: private data to be passed to @bio_ctr
3156  *
3157  * Description:
3158  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3159  *     Also, pages which the original bios are pointing to are not copied
3160  *     and the cloned bios just point same pages.
3161  *     So cloned bios must be completed before original bios, which means
3162  *     the caller must complete @rq before @rq_src.
3163  */
blk_rq_prep_clone(struct request * rq,struct request * rq_src,struct bio_set * bs,gfp_t gfp_mask,int (* bio_ctr)(struct bio *,struct bio *,void *),void * data)3164 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3165 		      struct bio_set *bs, gfp_t gfp_mask,
3166 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
3167 		      void *data)
3168 {
3169 	struct bio *bio, *bio_src;
3170 
3171 	if (!bs)
3172 		bs = &fs_bio_set;
3173 
3174 	__rq_for_each_bio(bio_src, rq_src) {
3175 		bio = bio_alloc_clone(rq->q->disk->part0, bio_src, gfp_mask,
3176 				      bs);
3177 		if (!bio)
3178 			goto free_and_out;
3179 
3180 		if (bio_ctr && bio_ctr(bio, bio_src, data))
3181 			goto free_and_out;
3182 
3183 		if (rq->bio) {
3184 			rq->biotail->bi_next = bio;
3185 			rq->biotail = bio;
3186 		} else {
3187 			rq->bio = rq->biotail = bio;
3188 		}
3189 		bio = NULL;
3190 	}
3191 
3192 	/* Copy attributes of the original request to the clone request. */
3193 	rq->__sector = blk_rq_pos(rq_src);
3194 	rq->__data_len = blk_rq_bytes(rq_src);
3195 	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3196 		rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
3197 		rq->special_vec = rq_src->special_vec;
3198 	}
3199 	rq->nr_phys_segments = rq_src->nr_phys_segments;
3200 	rq->ioprio = rq_src->ioprio;
3201 
3202 	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
3203 		goto free_and_out;
3204 
3205 	return 0;
3206 
3207 free_and_out:
3208 	if (bio)
3209 		bio_put(bio);
3210 	blk_rq_unprep_clone(rq);
3211 
3212 	return -ENOMEM;
3213 }
3214 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3215 #endif /* CONFIG_BLK_MQ_STACKING */
3216 
3217 /*
3218  * Steal bios from a request and add them to a bio list.
3219  * The request must not have been partially completed before.
3220  */
blk_steal_bios(struct bio_list * list,struct request * rq)3221 void blk_steal_bios(struct bio_list *list, struct request *rq)
3222 {
3223 	if (rq->bio) {
3224 		if (list->tail)
3225 			list->tail->bi_next = rq->bio;
3226 		else
3227 			list->head = rq->bio;
3228 		list->tail = rq->biotail;
3229 
3230 		rq->bio = NULL;
3231 		rq->biotail = NULL;
3232 	}
3233 
3234 	rq->__data_len = 0;
3235 }
3236 EXPORT_SYMBOL_GPL(blk_steal_bios);
3237 
order_to_size(unsigned int order)3238 static size_t order_to_size(unsigned int order)
3239 {
3240 	return (size_t)PAGE_SIZE << order;
3241 }
3242 
3243 /* called before freeing request pool in @tags */
blk_mq_clear_rq_mapping(struct blk_mq_tags * drv_tags,struct blk_mq_tags * tags)3244 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
3245 				    struct blk_mq_tags *tags)
3246 {
3247 	struct page *page;
3248 	unsigned long flags;
3249 
3250 	/*
3251 	 * There is no need to clear mapping if driver tags is not initialized
3252 	 * or the mapping belongs to the driver tags.
3253 	 */
3254 	if (!drv_tags || drv_tags == tags)
3255 		return;
3256 
3257 	list_for_each_entry(page, &tags->page_list, lru) {
3258 		unsigned long start = (unsigned long)page_address(page);
3259 		unsigned long end = start + order_to_size(page->private);
3260 		int i;
3261 
3262 		for (i = 0; i < drv_tags->nr_tags; i++) {
3263 			struct request *rq = drv_tags->rqs[i];
3264 			unsigned long rq_addr = (unsigned long)rq;
3265 
3266 			if (rq_addr >= start && rq_addr < end) {
3267 				WARN_ON_ONCE(req_ref_read(rq) != 0);
3268 				cmpxchg(&drv_tags->rqs[i], rq, NULL);
3269 			}
3270 		}
3271 	}
3272 
3273 	/*
3274 	 * Wait until all pending iteration is done.
3275 	 *
3276 	 * Request reference is cleared and it is guaranteed to be observed
3277 	 * after the ->lock is released.
3278 	 */
3279 	spin_lock_irqsave(&drv_tags->lock, flags);
3280 	spin_unlock_irqrestore(&drv_tags->lock, flags);
3281 }
3282 
blk_mq_free_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx)3283 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
3284 		     unsigned int hctx_idx)
3285 {
3286 	struct blk_mq_tags *drv_tags;
3287 	struct page *page;
3288 
3289 	if (list_empty(&tags->page_list))
3290 		return;
3291 
3292 	if (blk_mq_is_shared_tags(set->flags))
3293 		drv_tags = set->shared_tags;
3294 	else
3295 		drv_tags = set->tags[hctx_idx];
3296 
3297 	if (tags->static_rqs && set->ops->exit_request) {
3298 		int i;
3299 
3300 		for (i = 0; i < tags->nr_tags; i++) {
3301 			struct request *rq = tags->static_rqs[i];
3302 
3303 			if (!rq)
3304 				continue;
3305 			set->ops->exit_request(set, rq, hctx_idx);
3306 			tags->static_rqs[i] = NULL;
3307 		}
3308 	}
3309 
3310 	blk_mq_clear_rq_mapping(drv_tags, tags);
3311 
3312 	while (!list_empty(&tags->page_list)) {
3313 		page = list_first_entry(&tags->page_list, struct page, lru);
3314 		list_del_init(&page->lru);
3315 		/*
3316 		 * Remove kmemleak object previously allocated in
3317 		 * blk_mq_alloc_rqs().
3318 		 */
3319 		kmemleak_free(page_address(page));
3320 		__free_pages(page, page->private);
3321 	}
3322 }
3323 
blk_mq_free_rq_map(struct blk_mq_tags * tags)3324 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
3325 {
3326 	kfree(tags->rqs);
3327 	tags->rqs = NULL;
3328 	kfree(tags->static_rqs);
3329 	tags->static_rqs = NULL;
3330 
3331 	blk_mq_free_tags(tags);
3332 }
3333 
hctx_idx_to_type(struct blk_mq_tag_set * set,unsigned int hctx_idx)3334 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
3335 		unsigned int hctx_idx)
3336 {
3337 	int i;
3338 
3339 	for (i = 0; i < set->nr_maps; i++) {
3340 		unsigned int start = set->map[i].queue_offset;
3341 		unsigned int end = start + set->map[i].nr_queues;
3342 
3343 		if (hctx_idx >= start && hctx_idx < end)
3344 			break;
3345 	}
3346 
3347 	if (i >= set->nr_maps)
3348 		i = HCTX_TYPE_DEFAULT;
3349 
3350 	return i;
3351 }
3352 
blk_mq_get_hctx_node(struct blk_mq_tag_set * set,unsigned int hctx_idx)3353 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
3354 		unsigned int hctx_idx)
3355 {
3356 	enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
3357 
3358 	return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
3359 }
3360 
blk_mq_alloc_rq_map(struct blk_mq_tag_set * set,unsigned int hctx_idx,unsigned int nr_tags,unsigned int reserved_tags)3361 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
3362 					       unsigned int hctx_idx,
3363 					       unsigned int nr_tags,
3364 					       unsigned int reserved_tags)
3365 {
3366 	int node = blk_mq_get_hctx_node(set, hctx_idx);
3367 	struct blk_mq_tags *tags;
3368 
3369 	if (node == NUMA_NO_NODE)
3370 		node = set->numa_node;
3371 
3372 	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
3373 				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
3374 	if (!tags)
3375 		return NULL;
3376 
3377 	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3378 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3379 				 node);
3380 	if (!tags->rqs)
3381 		goto err_free_tags;
3382 
3383 	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3384 					GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3385 					node);
3386 	if (!tags->static_rqs)
3387 		goto err_free_rqs;
3388 
3389 	return tags;
3390 
3391 err_free_rqs:
3392 	kfree(tags->rqs);
3393 err_free_tags:
3394 	blk_mq_free_tags(tags);
3395 	return NULL;
3396 }
3397 
blk_mq_init_request(struct blk_mq_tag_set * set,struct request * rq,unsigned int hctx_idx,int node)3398 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
3399 			       unsigned int hctx_idx, int node)
3400 {
3401 	int ret;
3402 
3403 	if (set->ops->init_request) {
3404 		ret = set->ops->init_request(set, rq, hctx_idx, node);
3405 		if (ret)
3406 			return ret;
3407 	}
3408 
3409 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
3410 	return 0;
3411 }
3412 
blk_mq_alloc_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx,unsigned int depth)3413 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
3414 			    struct blk_mq_tags *tags,
3415 			    unsigned int hctx_idx, unsigned int depth)
3416 {
3417 	unsigned int i, j, entries_per_page, max_order = 4;
3418 	int node = blk_mq_get_hctx_node(set, hctx_idx);
3419 	size_t rq_size, left;
3420 
3421 	if (node == NUMA_NO_NODE)
3422 		node = set->numa_node;
3423 
3424 	INIT_LIST_HEAD(&tags->page_list);
3425 
3426 	/*
3427 	 * rq_size is the size of the request plus driver payload, rounded
3428 	 * to the cacheline size
3429 	 */
3430 	rq_size = round_up(sizeof(struct request) + set->cmd_size,
3431 				cache_line_size());
3432 	left = rq_size * depth;
3433 
3434 	for (i = 0; i < depth; ) {
3435 		int this_order = max_order;
3436 		struct page *page;
3437 		int to_do;
3438 		void *p;
3439 
3440 		while (this_order && left < order_to_size(this_order - 1))
3441 			this_order--;
3442 
3443 		do {
3444 			page = alloc_pages_node(node,
3445 				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
3446 				this_order);
3447 			if (page)
3448 				break;
3449 			if (!this_order--)
3450 				break;
3451 			if (order_to_size(this_order) < rq_size)
3452 				break;
3453 		} while (1);
3454 
3455 		if (!page)
3456 			goto fail;
3457 
3458 		page->private = this_order;
3459 		list_add_tail(&page->lru, &tags->page_list);
3460 
3461 		p = page_address(page);
3462 		/*
3463 		 * Allow kmemleak to scan these pages as they contain pointers
3464 		 * to additional allocations like via ops->init_request().
3465 		 */
3466 		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
3467 		entries_per_page = order_to_size(this_order) / rq_size;
3468 		to_do = min(entries_per_page, depth - i);
3469 		left -= to_do * rq_size;
3470 		for (j = 0; j < to_do; j++) {
3471 			struct request *rq = p;
3472 
3473 			tags->static_rqs[i] = rq;
3474 			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
3475 				tags->static_rqs[i] = NULL;
3476 				goto fail;
3477 			}
3478 
3479 			p += rq_size;
3480 			i++;
3481 		}
3482 	}
3483 	return 0;
3484 
3485 fail:
3486 	blk_mq_free_rqs(set, tags, hctx_idx);
3487 	return -ENOMEM;
3488 }
3489 
3490 struct rq_iter_data {
3491 	struct blk_mq_hw_ctx *hctx;
3492 	bool has_rq;
3493 };
3494 
blk_mq_has_request(struct request * rq,void * data)3495 static bool blk_mq_has_request(struct request *rq, void *data)
3496 {
3497 	struct rq_iter_data *iter_data = data;
3498 
3499 	if (rq->mq_hctx != iter_data->hctx)
3500 		return true;
3501 	iter_data->has_rq = true;
3502 	return false;
3503 }
3504 
blk_mq_hctx_has_requests(struct blk_mq_hw_ctx * hctx)3505 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
3506 {
3507 	struct blk_mq_tags *tags = hctx->sched_tags ?
3508 			hctx->sched_tags : hctx->tags;
3509 	struct rq_iter_data data = {
3510 		.hctx	= hctx,
3511 	};
3512 
3513 	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
3514 	return data.has_rq;
3515 }
3516 
blk_mq_last_cpu_in_hctx(unsigned int cpu,struct blk_mq_hw_ctx * hctx)3517 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
3518 		struct blk_mq_hw_ctx *hctx)
3519 {
3520 	if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu)
3521 		return false;
3522 	if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
3523 		return false;
3524 	return true;
3525 }
3526 
blk_mq_hctx_notify_offline(unsigned int cpu,struct hlist_node * node)3527 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
3528 {
3529 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3530 			struct blk_mq_hw_ctx, cpuhp_online);
3531 
3532 	if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
3533 	    !blk_mq_last_cpu_in_hctx(cpu, hctx))
3534 		return 0;
3535 
3536 	/*
3537 	 * Prevent new request from being allocated on the current hctx.
3538 	 *
3539 	 * The smp_mb__after_atomic() Pairs with the implied barrier in
3540 	 * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
3541 	 * seen once we return from the tag allocator.
3542 	 */
3543 	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3544 	smp_mb__after_atomic();
3545 
3546 	/*
3547 	 * Try to grab a reference to the queue and wait for any outstanding
3548 	 * requests.  If we could not grab a reference the queue has been
3549 	 * frozen and there are no requests.
3550 	 */
3551 	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
3552 		while (blk_mq_hctx_has_requests(hctx))
3553 			msleep(5);
3554 		percpu_ref_put(&hctx->queue->q_usage_counter);
3555 	}
3556 
3557 	return 0;
3558 }
3559 
blk_mq_hctx_notify_online(unsigned int cpu,struct hlist_node * node)3560 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
3561 {
3562 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3563 			struct blk_mq_hw_ctx, cpuhp_online);
3564 
3565 	if (cpumask_test_cpu(cpu, hctx->cpumask))
3566 		clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3567 	return 0;
3568 }
3569 
3570 /*
3571  * 'cpu' is going away. splice any existing rq_list entries from this
3572  * software queue to the hw queue dispatch list, and ensure that it
3573  * gets run.
3574  */
blk_mq_hctx_notify_dead(unsigned int cpu,struct hlist_node * node)3575 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
3576 {
3577 	struct blk_mq_hw_ctx *hctx;
3578 	struct blk_mq_ctx *ctx;
3579 	LIST_HEAD(tmp);
3580 	enum hctx_type type;
3581 
3582 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
3583 	if (!cpumask_test_cpu(cpu, hctx->cpumask))
3584 		return 0;
3585 
3586 	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
3587 	type = hctx->type;
3588 
3589 	spin_lock(&ctx->lock);
3590 	if (!list_empty(&ctx->rq_lists[type])) {
3591 		list_splice_init(&ctx->rq_lists[type], &tmp);
3592 		blk_mq_hctx_clear_pending(hctx, ctx);
3593 	}
3594 	spin_unlock(&ctx->lock);
3595 
3596 	if (list_empty(&tmp))
3597 		return 0;
3598 
3599 	spin_lock(&hctx->lock);
3600 	list_splice_tail_init(&tmp, &hctx->dispatch);
3601 	spin_unlock(&hctx->lock);
3602 
3603 	blk_mq_run_hw_queue(hctx, true);
3604 	return 0;
3605 }
3606 
blk_mq_remove_cpuhp(struct blk_mq_hw_ctx * hctx)3607 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3608 {
3609 	if (!(hctx->flags & BLK_MQ_F_STACKING))
3610 		cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3611 						    &hctx->cpuhp_online);
3612 	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3613 					    &hctx->cpuhp_dead);
3614 }
3615 
3616 /*
3617  * Before freeing hw queue, clearing the flush request reference in
3618  * tags->rqs[] for avoiding potential UAF.
3619  */
blk_mq_clear_flush_rq_mapping(struct blk_mq_tags * tags,unsigned int queue_depth,struct request * flush_rq)3620 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
3621 		unsigned int queue_depth, struct request *flush_rq)
3622 {
3623 	int i;
3624 	unsigned long flags;
3625 
3626 	/* The hw queue may not be mapped yet */
3627 	if (!tags)
3628 		return;
3629 
3630 	WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
3631 
3632 	for (i = 0; i < queue_depth; i++)
3633 		cmpxchg(&tags->rqs[i], flush_rq, NULL);
3634 
3635 	/*
3636 	 * Wait until all pending iteration is done.
3637 	 *
3638 	 * Request reference is cleared and it is guaranteed to be observed
3639 	 * after the ->lock is released.
3640 	 */
3641 	spin_lock_irqsave(&tags->lock, flags);
3642 	spin_unlock_irqrestore(&tags->lock, flags);
3643 }
3644 
3645 /* hctx->ctxs will be freed in queue's release handler */
blk_mq_exit_hctx(struct request_queue * q,struct blk_mq_tag_set * set,struct blk_mq_hw_ctx * hctx,unsigned int hctx_idx)3646 static void blk_mq_exit_hctx(struct request_queue *q,
3647 		struct blk_mq_tag_set *set,
3648 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
3649 {
3650 	struct request *flush_rq = hctx->fq->flush_rq;
3651 
3652 	if (blk_mq_hw_queue_mapped(hctx))
3653 		blk_mq_tag_idle(hctx);
3654 
3655 	if (blk_queue_init_done(q))
3656 		blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
3657 				set->queue_depth, flush_rq);
3658 	if (set->ops->exit_request)
3659 		set->ops->exit_request(set, flush_rq, hctx_idx);
3660 
3661 	if (set->ops->exit_hctx)
3662 		set->ops->exit_hctx(hctx, hctx_idx);
3663 
3664 	blk_mq_remove_cpuhp(hctx);
3665 
3666 	xa_erase(&q->hctx_table, hctx_idx);
3667 
3668 	spin_lock(&q->unused_hctx_lock);
3669 	list_add(&hctx->hctx_list, &q->unused_hctx_list);
3670 	spin_unlock(&q->unused_hctx_lock);
3671 }
3672 
blk_mq_exit_hw_queues(struct request_queue * q,struct blk_mq_tag_set * set,int nr_queue)3673 static void blk_mq_exit_hw_queues(struct request_queue *q,
3674 		struct blk_mq_tag_set *set, int nr_queue)
3675 {
3676 	struct blk_mq_hw_ctx *hctx;
3677 	unsigned long i;
3678 
3679 	queue_for_each_hw_ctx(q, hctx, i) {
3680 		if (i == nr_queue)
3681 			break;
3682 		blk_mq_exit_hctx(q, set, hctx, i);
3683 	}
3684 }
3685 
blk_mq_init_hctx(struct request_queue * q,struct blk_mq_tag_set * set,struct blk_mq_hw_ctx * hctx,unsigned hctx_idx)3686 static int blk_mq_init_hctx(struct request_queue *q,
3687 		struct blk_mq_tag_set *set,
3688 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
3689 {
3690 	hctx->queue_num = hctx_idx;
3691 
3692 	if (!(hctx->flags & BLK_MQ_F_STACKING))
3693 		cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3694 				&hctx->cpuhp_online);
3695 	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
3696 
3697 	hctx->tags = set->tags[hctx_idx];
3698 
3699 	if (set->ops->init_hctx &&
3700 	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
3701 		goto unregister_cpu_notifier;
3702 
3703 	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
3704 				hctx->numa_node))
3705 		goto exit_hctx;
3706 
3707 	if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
3708 		goto exit_flush_rq;
3709 
3710 	return 0;
3711 
3712  exit_flush_rq:
3713 	if (set->ops->exit_request)
3714 		set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
3715  exit_hctx:
3716 	if (set->ops->exit_hctx)
3717 		set->ops->exit_hctx(hctx, hctx_idx);
3718  unregister_cpu_notifier:
3719 	blk_mq_remove_cpuhp(hctx);
3720 	return -1;
3721 }
3722 
3723 static struct blk_mq_hw_ctx *
blk_mq_alloc_hctx(struct request_queue * q,struct blk_mq_tag_set * set,int node)3724 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
3725 		int node)
3726 {
3727 	struct blk_mq_hw_ctx *hctx;
3728 	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
3729 
3730 	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
3731 	if (!hctx)
3732 		goto fail_alloc_hctx;
3733 
3734 	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
3735 		goto free_hctx;
3736 
3737 	atomic_set(&hctx->nr_active, 0);
3738 	if (node == NUMA_NO_NODE)
3739 		node = set->numa_node;
3740 	hctx->numa_node = node;
3741 
3742 	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
3743 	spin_lock_init(&hctx->lock);
3744 	INIT_LIST_HEAD(&hctx->dispatch);
3745 	hctx->queue = q;
3746 	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
3747 
3748 	INIT_LIST_HEAD(&hctx->hctx_list);
3749 
3750 	/*
3751 	 * Allocate space for all possible cpus to avoid allocation at
3752 	 * runtime
3753 	 */
3754 	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
3755 			gfp, node);
3756 	if (!hctx->ctxs)
3757 		goto free_cpumask;
3758 
3759 	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
3760 				gfp, node, false, false))
3761 		goto free_ctxs;
3762 	hctx->nr_ctx = 0;
3763 
3764 	spin_lock_init(&hctx->dispatch_wait_lock);
3765 	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
3766 	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
3767 
3768 	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
3769 	if (!hctx->fq)
3770 		goto free_bitmap;
3771 
3772 	blk_mq_hctx_kobj_init(hctx);
3773 
3774 	return hctx;
3775 
3776  free_bitmap:
3777 	sbitmap_free(&hctx->ctx_map);
3778  free_ctxs:
3779 	kfree(hctx->ctxs);
3780  free_cpumask:
3781 	free_cpumask_var(hctx->cpumask);
3782  free_hctx:
3783 	kfree(hctx);
3784  fail_alloc_hctx:
3785 	return NULL;
3786 }
3787 
blk_mq_init_cpu_queues(struct request_queue * q,unsigned int nr_hw_queues)3788 static void blk_mq_init_cpu_queues(struct request_queue *q,
3789 				   unsigned int nr_hw_queues)
3790 {
3791 	struct blk_mq_tag_set *set = q->tag_set;
3792 	unsigned int i, j;
3793 
3794 	for_each_possible_cpu(i) {
3795 		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
3796 		struct blk_mq_hw_ctx *hctx;
3797 		int k;
3798 
3799 		__ctx->cpu = i;
3800 		spin_lock_init(&__ctx->lock);
3801 		for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
3802 			INIT_LIST_HEAD(&__ctx->rq_lists[k]);
3803 
3804 		__ctx->queue = q;
3805 
3806 		/*
3807 		 * Set local node, IFF we have more than one hw queue. If
3808 		 * not, we remain on the home node of the device
3809 		 */
3810 		for (j = 0; j < set->nr_maps; j++) {
3811 			hctx = blk_mq_map_queue_type(q, j, i);
3812 			if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
3813 				hctx->numa_node = cpu_to_node(i);
3814 		}
3815 	}
3816 }
3817 
blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set * set,unsigned int hctx_idx,unsigned int depth)3818 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
3819 					     unsigned int hctx_idx,
3820 					     unsigned int depth)
3821 {
3822 	struct blk_mq_tags *tags;
3823 	int ret;
3824 
3825 	tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
3826 	if (!tags)
3827 		return NULL;
3828 
3829 	ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
3830 	if (ret) {
3831 		blk_mq_free_rq_map(tags);
3832 		return NULL;
3833 	}
3834 
3835 	return tags;
3836 }
3837 
__blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set * set,int hctx_idx)3838 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
3839 				       int hctx_idx)
3840 {
3841 	if (blk_mq_is_shared_tags(set->flags)) {
3842 		set->tags[hctx_idx] = set->shared_tags;
3843 
3844 		return true;
3845 	}
3846 
3847 	set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
3848 						       set->queue_depth);
3849 
3850 	return set->tags[hctx_idx];
3851 }
3852 
blk_mq_free_map_and_rqs(struct blk_mq_tag_set * set,struct blk_mq_tags * tags,unsigned int hctx_idx)3853 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
3854 			     struct blk_mq_tags *tags,
3855 			     unsigned int hctx_idx)
3856 {
3857 	if (tags) {
3858 		blk_mq_free_rqs(set, tags, hctx_idx);
3859 		blk_mq_free_rq_map(tags);
3860 	}
3861 }
3862 
__blk_mq_free_map_and_rqs(struct blk_mq_tag_set * set,unsigned int hctx_idx)3863 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
3864 				      unsigned int hctx_idx)
3865 {
3866 	if (!blk_mq_is_shared_tags(set->flags))
3867 		blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
3868 
3869 	set->tags[hctx_idx] = NULL;
3870 }
3871 
blk_mq_map_swqueue(struct request_queue * q)3872 static void blk_mq_map_swqueue(struct request_queue *q)
3873 {
3874 	unsigned int j, hctx_idx;
3875 	unsigned long i;
3876 	struct blk_mq_hw_ctx *hctx;
3877 	struct blk_mq_ctx *ctx;
3878 	struct blk_mq_tag_set *set = q->tag_set;
3879 
3880 	queue_for_each_hw_ctx(q, hctx, i) {
3881 		cpumask_clear(hctx->cpumask);
3882 		hctx->nr_ctx = 0;
3883 		hctx->dispatch_from = NULL;
3884 	}
3885 
3886 	/*
3887 	 * Map software to hardware queues.
3888 	 *
3889 	 * If the cpu isn't present, the cpu is mapped to first hctx.
3890 	 */
3891 	for_each_possible_cpu(i) {
3892 
3893 		ctx = per_cpu_ptr(q->queue_ctx, i);
3894 		for (j = 0; j < set->nr_maps; j++) {
3895 			if (!set->map[j].nr_queues) {
3896 				ctx->hctxs[j] = blk_mq_map_queue_type(q,
3897 						HCTX_TYPE_DEFAULT, i);
3898 				continue;
3899 			}
3900 			hctx_idx = set->map[j].mq_map[i];
3901 			/* unmapped hw queue can be remapped after CPU topo changed */
3902 			if (!set->tags[hctx_idx] &&
3903 			    !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
3904 				/*
3905 				 * If tags initialization fail for some hctx,
3906 				 * that hctx won't be brought online.  In this
3907 				 * case, remap the current ctx to hctx[0] which
3908 				 * is guaranteed to always have tags allocated
3909 				 */
3910 				set->map[j].mq_map[i] = 0;
3911 			}
3912 
3913 			hctx = blk_mq_map_queue_type(q, j, i);
3914 			ctx->hctxs[j] = hctx;
3915 			/*
3916 			 * If the CPU is already set in the mask, then we've
3917 			 * mapped this one already. This can happen if
3918 			 * devices share queues across queue maps.
3919 			 */
3920 			if (cpumask_test_cpu(i, hctx->cpumask))
3921 				continue;
3922 
3923 			cpumask_set_cpu(i, hctx->cpumask);
3924 			hctx->type = j;
3925 			ctx->index_hw[hctx->type] = hctx->nr_ctx;
3926 			hctx->ctxs[hctx->nr_ctx++] = ctx;
3927 
3928 			/*
3929 			 * If the nr_ctx type overflows, we have exceeded the
3930 			 * amount of sw queues we can support.
3931 			 */
3932 			BUG_ON(!hctx->nr_ctx);
3933 		}
3934 
3935 		for (; j < HCTX_MAX_TYPES; j++)
3936 			ctx->hctxs[j] = blk_mq_map_queue_type(q,
3937 					HCTX_TYPE_DEFAULT, i);
3938 	}
3939 
3940 	queue_for_each_hw_ctx(q, hctx, i) {
3941 		/*
3942 		 * If no software queues are mapped to this hardware queue,
3943 		 * disable it and free the request entries.
3944 		 */
3945 		if (!hctx->nr_ctx) {
3946 			/* Never unmap queue 0.  We need it as a
3947 			 * fallback in case of a new remap fails
3948 			 * allocation
3949 			 */
3950 			if (i)
3951 				__blk_mq_free_map_and_rqs(set, i);
3952 
3953 			hctx->tags = NULL;
3954 			continue;
3955 		}
3956 
3957 		hctx->tags = set->tags[i];
3958 		WARN_ON(!hctx->tags);
3959 
3960 		/*
3961 		 * Set the map size to the number of mapped software queues.
3962 		 * This is more accurate and more efficient than looping
3963 		 * over all possibly mapped software queues.
3964 		 */
3965 		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
3966 
3967 		/*
3968 		 * Initialize batch roundrobin counts
3969 		 */
3970 		hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
3971 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
3972 	}
3973 }
3974 
3975 /*
3976  * Caller needs to ensure that we're either frozen/quiesced, or that
3977  * the queue isn't live yet.
3978  */
queue_set_hctx_shared(struct request_queue * q,bool shared)3979 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
3980 {
3981 	struct blk_mq_hw_ctx *hctx;
3982 	unsigned long i;
3983 
3984 	queue_for_each_hw_ctx(q, hctx, i) {
3985 		if (shared) {
3986 			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
3987 		} else {
3988 			blk_mq_tag_idle(hctx);
3989 			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
3990 		}
3991 	}
3992 }
3993 
blk_mq_update_tag_set_shared(struct blk_mq_tag_set * set,bool shared)3994 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
3995 					 bool shared)
3996 {
3997 	struct request_queue *q;
3998 
3999 	lockdep_assert_held(&set->tag_list_lock);
4000 
4001 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4002 		blk_mq_freeze_queue(q);
4003 		queue_set_hctx_shared(q, shared);
4004 		blk_mq_unfreeze_queue(q);
4005 	}
4006 }
4007 
blk_mq_del_queue_tag_set(struct request_queue * q)4008 static void blk_mq_del_queue_tag_set(struct request_queue *q)
4009 {
4010 	struct blk_mq_tag_set *set = q->tag_set;
4011 
4012 	mutex_lock(&set->tag_list_lock);
4013 	list_del(&q->tag_set_list);
4014 	if (list_is_singular(&set->tag_list)) {
4015 		/* just transitioned to unshared */
4016 		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4017 		/* update existing queue */
4018 		blk_mq_update_tag_set_shared(set, false);
4019 	}
4020 	mutex_unlock(&set->tag_list_lock);
4021 	INIT_LIST_HEAD(&q->tag_set_list);
4022 }
4023 
blk_mq_add_queue_tag_set(struct blk_mq_tag_set * set,struct request_queue * q)4024 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4025 				     struct request_queue *q)
4026 {
4027 	mutex_lock(&set->tag_list_lock);
4028 
4029 	/*
4030 	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
4031 	 */
4032 	if (!list_empty(&set->tag_list) &&
4033 	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
4034 		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4035 		/* update existing queue */
4036 		blk_mq_update_tag_set_shared(set, true);
4037 	}
4038 	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4039 		queue_set_hctx_shared(q, true);
4040 	list_add_tail(&q->tag_set_list, &set->tag_list);
4041 
4042 	mutex_unlock(&set->tag_list_lock);
4043 }
4044 
4045 /* All allocations will be freed in release handler of q->mq_kobj */
blk_mq_alloc_ctxs(struct request_queue * q)4046 static int blk_mq_alloc_ctxs(struct request_queue *q)
4047 {
4048 	struct blk_mq_ctxs *ctxs;
4049 	int cpu;
4050 
4051 	ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
4052 	if (!ctxs)
4053 		return -ENOMEM;
4054 
4055 	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
4056 	if (!ctxs->queue_ctx)
4057 		goto fail;
4058 
4059 	for_each_possible_cpu(cpu) {
4060 		struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
4061 		ctx->ctxs = ctxs;
4062 	}
4063 
4064 	q->mq_kobj = &ctxs->kobj;
4065 	q->queue_ctx = ctxs->queue_ctx;
4066 
4067 	return 0;
4068  fail:
4069 	kfree(ctxs);
4070 	return -ENOMEM;
4071 }
4072 
4073 /*
4074  * It is the actual release handler for mq, but we do it from
4075  * request queue's release handler for avoiding use-after-free
4076  * and headache because q->mq_kobj shouldn't have been introduced,
4077  * but we can't group ctx/kctx kobj without it.
4078  */
blk_mq_release(struct request_queue * q)4079 void blk_mq_release(struct request_queue *q)
4080 {
4081 	struct blk_mq_hw_ctx *hctx, *next;
4082 	unsigned long i;
4083 
4084 	queue_for_each_hw_ctx(q, hctx, i)
4085 		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
4086 
4087 	/* all hctx are in .unused_hctx_list now */
4088 	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
4089 		list_del_init(&hctx->hctx_list);
4090 		kobject_put(&hctx->kobj);
4091 	}
4092 
4093 	xa_destroy(&q->hctx_table);
4094 
4095 	/*
4096 	 * release .mq_kobj and sw queue's kobject now because
4097 	 * both share lifetime with request queue.
4098 	 */
4099 	blk_mq_sysfs_deinit(q);
4100 }
4101 
blk_mq_init_queue_data(struct blk_mq_tag_set * set,void * queuedata)4102 static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
4103 		void *queuedata)
4104 {
4105 	struct request_queue *q;
4106 	int ret;
4107 
4108 	q = blk_alloc_queue(set->numa_node);
4109 	if (!q)
4110 		return ERR_PTR(-ENOMEM);
4111 	q->queuedata = queuedata;
4112 	ret = blk_mq_init_allocated_queue(set, q);
4113 	if (ret) {
4114 		blk_put_queue(q);
4115 		return ERR_PTR(ret);
4116 	}
4117 	return q;
4118 }
4119 
blk_mq_init_queue(struct blk_mq_tag_set * set)4120 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
4121 {
4122 	return blk_mq_init_queue_data(set, NULL);
4123 }
4124 EXPORT_SYMBOL(blk_mq_init_queue);
4125 
4126 /**
4127  * blk_mq_destroy_queue - shutdown a request queue
4128  * @q: request queue to shutdown
4129  *
4130  * This shuts down a request queue allocated by blk_mq_init_queue(). All future
4131  * requests will be failed with -ENODEV. The caller is responsible for dropping
4132  * the reference from blk_mq_init_queue() by calling blk_put_queue().
4133  *
4134  * Context: can sleep
4135  */
blk_mq_destroy_queue(struct request_queue * q)4136 void blk_mq_destroy_queue(struct request_queue *q)
4137 {
4138 	WARN_ON_ONCE(!queue_is_mq(q));
4139 	WARN_ON_ONCE(blk_queue_registered(q));
4140 
4141 	might_sleep();
4142 
4143 	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
4144 	blk_queue_start_drain(q);
4145 	blk_mq_freeze_queue_wait(q);
4146 
4147 	blk_sync_queue(q);
4148 	blk_mq_cancel_work_sync(q);
4149 	blk_mq_exit_queue(q);
4150 }
4151 EXPORT_SYMBOL(blk_mq_destroy_queue);
4152 
__blk_mq_alloc_disk(struct blk_mq_tag_set * set,void * queuedata,struct lock_class_key * lkclass)4153 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
4154 		struct lock_class_key *lkclass)
4155 {
4156 	struct request_queue *q;
4157 	struct gendisk *disk;
4158 
4159 	q = blk_mq_init_queue_data(set, queuedata);
4160 	if (IS_ERR(q))
4161 		return ERR_CAST(q);
4162 
4163 	disk = __alloc_disk_node(q, set->numa_node, lkclass);
4164 	if (!disk) {
4165 		blk_mq_destroy_queue(q);
4166 		blk_put_queue(q);
4167 		return ERR_PTR(-ENOMEM);
4168 	}
4169 	set_bit(GD_OWNS_QUEUE, &disk->state);
4170 	return disk;
4171 }
4172 EXPORT_SYMBOL(__blk_mq_alloc_disk);
4173 
blk_mq_alloc_disk_for_queue(struct request_queue * q,struct lock_class_key * lkclass)4174 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
4175 		struct lock_class_key *lkclass)
4176 {
4177 	struct gendisk *disk;
4178 
4179 	if (!blk_get_queue(q))
4180 		return NULL;
4181 	disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
4182 	if (!disk)
4183 		blk_put_queue(q);
4184 	return disk;
4185 }
4186 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
4187 
blk_mq_alloc_and_init_hctx(struct blk_mq_tag_set * set,struct request_queue * q,int hctx_idx,int node)4188 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
4189 		struct blk_mq_tag_set *set, struct request_queue *q,
4190 		int hctx_idx, int node)
4191 {
4192 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;
4193 
4194 	/* reuse dead hctx first */
4195 	spin_lock(&q->unused_hctx_lock);
4196 	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
4197 		if (tmp->numa_node == node) {
4198 			hctx = tmp;
4199 			break;
4200 		}
4201 	}
4202 	if (hctx)
4203 		list_del_init(&hctx->hctx_list);
4204 	spin_unlock(&q->unused_hctx_lock);
4205 
4206 	if (!hctx)
4207 		hctx = blk_mq_alloc_hctx(q, set, node);
4208 	if (!hctx)
4209 		goto fail;
4210 
4211 	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
4212 		goto free_hctx;
4213 
4214 	return hctx;
4215 
4216  free_hctx:
4217 	kobject_put(&hctx->kobj);
4218  fail:
4219 	return NULL;
4220 }
4221 
blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set * set,struct request_queue * q)4222 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4223 						struct request_queue *q)
4224 {
4225 	struct blk_mq_hw_ctx *hctx;
4226 	unsigned long i, j;
4227 
4228 	/* protect against switching io scheduler  */
4229 	mutex_lock(&q->sysfs_lock);
4230 	for (i = 0; i < set->nr_hw_queues; i++) {
4231 		int old_node;
4232 		int node = blk_mq_get_hctx_node(set, i);
4233 		struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);
4234 
4235 		if (old_hctx) {
4236 			old_node = old_hctx->numa_node;
4237 			blk_mq_exit_hctx(q, set, old_hctx, i);
4238 		}
4239 
4240 		if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) {
4241 			if (!old_hctx)
4242 				break;
4243 			pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
4244 					node, old_node);
4245 			hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node);
4246 			WARN_ON_ONCE(!hctx);
4247 		}
4248 	}
4249 	/*
4250 	 * Increasing nr_hw_queues fails. Free the newly allocated
4251 	 * hctxs and keep the previous q->nr_hw_queues.
4252 	 */
4253 	if (i != set->nr_hw_queues) {
4254 		j = q->nr_hw_queues;
4255 	} else {
4256 		j = i;
4257 		q->nr_hw_queues = set->nr_hw_queues;
4258 	}
4259 
4260 	xa_for_each_start(&q->hctx_table, j, hctx, j)
4261 		blk_mq_exit_hctx(q, set, hctx, j);
4262 	mutex_unlock(&q->sysfs_lock);
4263 }
4264 
blk_mq_update_poll_flag(struct request_queue * q)4265 static void blk_mq_update_poll_flag(struct request_queue *q)
4266 {
4267 	struct blk_mq_tag_set *set = q->tag_set;
4268 
4269 	if (set->nr_maps > HCTX_TYPE_POLL &&
4270 	    set->map[HCTX_TYPE_POLL].nr_queues)
4271 		blk_queue_flag_set(QUEUE_FLAG_POLL, q);
4272 	else
4273 		blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
4274 }
4275 
blk_mq_init_allocated_queue(struct blk_mq_tag_set * set,struct request_queue * q)4276 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
4277 		struct request_queue *q)
4278 {
4279 	/* mark the queue as mq asap */
4280 	q->mq_ops = set->ops;
4281 
4282 	if (blk_mq_alloc_ctxs(q))
4283 		goto err_exit;
4284 
4285 	/* init q->mq_kobj and sw queues' kobjects */
4286 	blk_mq_sysfs_init(q);
4287 
4288 	INIT_LIST_HEAD(&q->unused_hctx_list);
4289 	spin_lock_init(&q->unused_hctx_lock);
4290 
4291 	xa_init(&q->hctx_table);
4292 
4293 	blk_mq_realloc_hw_ctxs(set, q);
4294 	if (!q->nr_hw_queues)
4295 		goto err_hctxs;
4296 
4297 	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
4298 	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
4299 
4300 	q->tag_set = set;
4301 
4302 	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
4303 	blk_mq_update_poll_flag(q);
4304 
4305 	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
4306 	INIT_LIST_HEAD(&q->flush_list);
4307 	INIT_LIST_HEAD(&q->requeue_list);
4308 	spin_lock_init(&q->requeue_lock);
4309 
4310 	q->nr_requests = set->queue_depth;
4311 
4312 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
4313 	blk_mq_add_queue_tag_set(set, q);
4314 	blk_mq_map_swqueue(q);
4315 	return 0;
4316 
4317 err_hctxs:
4318 	blk_mq_release(q);
4319 err_exit:
4320 	q->mq_ops = NULL;
4321 	return -ENOMEM;
4322 }
4323 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
4324 
4325 /* tags can _not_ be used after returning from blk_mq_exit_queue */
blk_mq_exit_queue(struct request_queue * q)4326 void blk_mq_exit_queue(struct request_queue *q)
4327 {
4328 	struct blk_mq_tag_set *set = q->tag_set;
4329 
4330 	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
4331 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
4332 	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
4333 	blk_mq_del_queue_tag_set(q);
4334 }
4335 
__blk_mq_alloc_rq_maps(struct blk_mq_tag_set * set)4336 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
4337 {
4338 	int i;
4339 
4340 	if (blk_mq_is_shared_tags(set->flags)) {
4341 		set->shared_tags = blk_mq_alloc_map_and_rqs(set,
4342 						BLK_MQ_NO_HCTX_IDX,
4343 						set->queue_depth);
4344 		if (!set->shared_tags)
4345 			return -ENOMEM;
4346 	}
4347 
4348 	for (i = 0; i < set->nr_hw_queues; i++) {
4349 		if (!__blk_mq_alloc_map_and_rqs(set, i))
4350 			goto out_unwind;
4351 		cond_resched();
4352 	}
4353 
4354 	return 0;
4355 
4356 out_unwind:
4357 	while (--i >= 0)
4358 		__blk_mq_free_map_and_rqs(set, i);
4359 
4360 	if (blk_mq_is_shared_tags(set->flags)) {
4361 		blk_mq_free_map_and_rqs(set, set->shared_tags,
4362 					BLK_MQ_NO_HCTX_IDX);
4363 	}
4364 
4365 	return -ENOMEM;
4366 }
4367 
4368 /*
4369  * Allocate the request maps associated with this tag_set. Note that this
4370  * may reduce the depth asked for, if memory is tight. set->queue_depth
4371  * will be updated to reflect the allocated depth.
4372  */
blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set * set)4373 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
4374 {
4375 	unsigned int depth;
4376 	int err;
4377 
4378 	depth = set->queue_depth;
4379 	do {
4380 		err = __blk_mq_alloc_rq_maps(set);
4381 		if (!err)
4382 			break;
4383 
4384 		set->queue_depth >>= 1;
4385 		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
4386 			err = -ENOMEM;
4387 			break;
4388 		}
4389 	} while (set->queue_depth);
4390 
4391 	if (!set->queue_depth || err) {
4392 		pr_err("blk-mq: failed to allocate request map\n");
4393 		return -ENOMEM;
4394 	}
4395 
4396 	if (depth != set->queue_depth)
4397 		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
4398 						depth, set->queue_depth);
4399 
4400 	return 0;
4401 }
4402 
blk_mq_update_queue_map(struct blk_mq_tag_set * set)4403 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
4404 {
4405 	/*
4406 	 * blk_mq_map_queues() and multiple .map_queues() implementations
4407 	 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
4408 	 * number of hardware queues.
4409 	 */
4410 	if (set->nr_maps == 1)
4411 		set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
4412 
4413 	if (set->ops->map_queues && !is_kdump_kernel()) {
4414 		int i;
4415 
4416 		/*
4417 		 * transport .map_queues is usually done in the following
4418 		 * way:
4419 		 *
4420 		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
4421 		 * 	mask = get_cpu_mask(queue)
4422 		 * 	for_each_cpu(cpu, mask)
4423 		 * 		set->map[x].mq_map[cpu] = queue;
4424 		 * }
4425 		 *
4426 		 * When we need to remap, the table has to be cleared for
4427 		 * killing stale mapping since one CPU may not be mapped
4428 		 * to any hw queue.
4429 		 */
4430 		for (i = 0; i < set->nr_maps; i++)
4431 			blk_mq_clear_mq_map(&set->map[i]);
4432 
4433 		set->ops->map_queues(set);
4434 	} else {
4435 		BUG_ON(set->nr_maps > 1);
4436 		blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
4437 	}
4438 }
4439 
blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set * set,int new_nr_hw_queues)4440 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
4441 				       int new_nr_hw_queues)
4442 {
4443 	struct blk_mq_tags **new_tags;
4444 	int i;
4445 
4446 	if (set->nr_hw_queues >= new_nr_hw_queues)
4447 		goto done;
4448 
4449 	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
4450 				GFP_KERNEL, set->numa_node);
4451 	if (!new_tags)
4452 		return -ENOMEM;
4453 
4454 	if (set->tags)
4455 		memcpy(new_tags, set->tags, set->nr_hw_queues *
4456 		       sizeof(*set->tags));
4457 	kfree(set->tags);
4458 	set->tags = new_tags;
4459 
4460 	for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
4461 		if (!__blk_mq_alloc_map_and_rqs(set, i)) {
4462 			while (--i >= set->nr_hw_queues)
4463 				__blk_mq_free_map_and_rqs(set, i);
4464 			return -ENOMEM;
4465 		}
4466 		cond_resched();
4467 	}
4468 
4469 done:
4470 	set->nr_hw_queues = new_nr_hw_queues;
4471 	return 0;
4472 }
4473 
4474 /*
4475  * Alloc a tag set to be associated with one or more request queues.
4476  * May fail with EINVAL for various error conditions. May adjust the
4477  * requested depth down, if it's too large. In that case, the set
4478  * value will be stored in set->queue_depth.
4479  */
blk_mq_alloc_tag_set(struct blk_mq_tag_set * set)4480 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
4481 {
4482 	int i, ret;
4483 
4484 	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
4485 
4486 	if (!set->nr_hw_queues)
4487 		return -EINVAL;
4488 	if (!set->queue_depth)
4489 		return -EINVAL;
4490 	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
4491 		return -EINVAL;
4492 
4493 	if (!set->ops->queue_rq)
4494 		return -EINVAL;
4495 
4496 	if (!set->ops->get_budget ^ !set->ops->put_budget)
4497 		return -EINVAL;
4498 
4499 	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
4500 		pr_info("blk-mq: reduced tag depth to %u\n",
4501 			BLK_MQ_MAX_DEPTH);
4502 		set->queue_depth = BLK_MQ_MAX_DEPTH;
4503 	}
4504 
4505 	if (!set->nr_maps)
4506 		set->nr_maps = 1;
4507 	else if (set->nr_maps > HCTX_MAX_TYPES)
4508 		return -EINVAL;
4509 
4510 	/*
4511 	 * If a crashdump is active, then we are potentially in a very
4512 	 * memory constrained environment. Limit us to 1 queue and
4513 	 * 64 tags to prevent using too much memory.
4514 	 */
4515 	if (is_kdump_kernel()) {
4516 		set->nr_hw_queues = 1;
4517 		set->nr_maps = 1;
4518 		set->queue_depth = min(64U, set->queue_depth);
4519 	}
4520 	/*
4521 	 * There is no use for more h/w queues than cpus if we just have
4522 	 * a single map
4523 	 */
4524 	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
4525 		set->nr_hw_queues = nr_cpu_ids;
4526 
4527 	if (set->flags & BLK_MQ_F_BLOCKING) {
4528 		set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL);
4529 		if (!set->srcu)
4530 			return -ENOMEM;
4531 		ret = init_srcu_struct(set->srcu);
4532 		if (ret)
4533 			goto out_free_srcu;
4534 	}
4535 
4536 	ret = -ENOMEM;
4537 	set->tags = kcalloc_node(set->nr_hw_queues,
4538 				 sizeof(struct blk_mq_tags *), GFP_KERNEL,
4539 				 set->numa_node);
4540 	if (!set->tags)
4541 		goto out_cleanup_srcu;
4542 
4543 	for (i = 0; i < set->nr_maps; i++) {
4544 		set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
4545 						  sizeof(set->map[i].mq_map[0]),
4546 						  GFP_KERNEL, set->numa_node);
4547 		if (!set->map[i].mq_map)
4548 			goto out_free_mq_map;
4549 		set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
4550 	}
4551 
4552 	blk_mq_update_queue_map(set);
4553 
4554 	ret = blk_mq_alloc_set_map_and_rqs(set);
4555 	if (ret)
4556 		goto out_free_mq_map;
4557 
4558 	mutex_init(&set->tag_list_lock);
4559 	INIT_LIST_HEAD(&set->tag_list);
4560 
4561 	return 0;
4562 
4563 out_free_mq_map:
4564 	for (i = 0; i < set->nr_maps; i++) {
4565 		kfree(set->map[i].mq_map);
4566 		set->map[i].mq_map = NULL;
4567 	}
4568 	kfree(set->tags);
4569 	set->tags = NULL;
4570 out_cleanup_srcu:
4571 	if (set->flags & BLK_MQ_F_BLOCKING)
4572 		cleanup_srcu_struct(set->srcu);
4573 out_free_srcu:
4574 	if (set->flags & BLK_MQ_F_BLOCKING)
4575 		kfree(set->srcu);
4576 	return ret;
4577 }
4578 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
4579 
4580 /* allocate and initialize a tagset for a simple single-queue device */
blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set * set,const struct blk_mq_ops * ops,unsigned int queue_depth,unsigned int set_flags)4581 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
4582 		const struct blk_mq_ops *ops, unsigned int queue_depth,
4583 		unsigned int set_flags)
4584 {
4585 	memset(set, 0, sizeof(*set));
4586 	set->ops = ops;
4587 	set->nr_hw_queues = 1;
4588 	set->nr_maps = 1;
4589 	set->queue_depth = queue_depth;
4590 	set->numa_node = NUMA_NO_NODE;
4591 	set->flags = set_flags;
4592 	return blk_mq_alloc_tag_set(set);
4593 }
4594 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
4595 
blk_mq_free_tag_set(struct blk_mq_tag_set * set)4596 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
4597 {
4598 	int i, j;
4599 
4600 	for (i = 0; i < set->nr_hw_queues; i++)
4601 		__blk_mq_free_map_and_rqs(set, i);
4602 
4603 	if (blk_mq_is_shared_tags(set->flags)) {
4604 		blk_mq_free_map_and_rqs(set, set->shared_tags,
4605 					BLK_MQ_NO_HCTX_IDX);
4606 	}
4607 
4608 	for (j = 0; j < set->nr_maps; j++) {
4609 		kfree(set->map[j].mq_map);
4610 		set->map[j].mq_map = NULL;
4611 	}
4612 
4613 	kfree(set->tags);
4614 	set->tags = NULL;
4615 	if (set->flags & BLK_MQ_F_BLOCKING) {
4616 		cleanup_srcu_struct(set->srcu);
4617 		kfree(set->srcu);
4618 	}
4619 }
4620 EXPORT_SYMBOL(blk_mq_free_tag_set);
4621 
blk_mq_update_nr_requests(struct request_queue * q,unsigned int nr)4622 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
4623 {
4624 	struct blk_mq_tag_set *set = q->tag_set;
4625 	struct blk_mq_hw_ctx *hctx;
4626 	int ret;
4627 	unsigned long i;
4628 
4629 	if (!set)
4630 		return -EINVAL;
4631 
4632 	if (q->nr_requests == nr)
4633 		return 0;
4634 
4635 	blk_mq_freeze_queue(q);
4636 	blk_mq_quiesce_queue(q);
4637 
4638 	ret = 0;
4639 	queue_for_each_hw_ctx(q, hctx, i) {
4640 		if (!hctx->tags)
4641 			continue;
4642 		/*
4643 		 * If we're using an MQ scheduler, just update the scheduler
4644 		 * queue depth. This is similar to what the old code would do.
4645 		 */
4646 		if (hctx->sched_tags) {
4647 			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
4648 						      nr, true);
4649 		} else {
4650 			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
4651 						      false);
4652 		}
4653 		if (ret)
4654 			break;
4655 		if (q->elevator && q->elevator->type->ops.depth_updated)
4656 			q->elevator->type->ops.depth_updated(hctx);
4657 	}
4658 	if (!ret) {
4659 		q->nr_requests = nr;
4660 		if (blk_mq_is_shared_tags(set->flags)) {
4661 			if (q->elevator)
4662 				blk_mq_tag_update_sched_shared_tags(q);
4663 			else
4664 				blk_mq_tag_resize_shared_tags(set, nr);
4665 		}
4666 	}
4667 
4668 	blk_mq_unquiesce_queue(q);
4669 	blk_mq_unfreeze_queue(q);
4670 
4671 	return ret;
4672 }
4673 
4674 /*
4675  * request_queue and elevator_type pair.
4676  * It is just used by __blk_mq_update_nr_hw_queues to cache
4677  * the elevator_type associated with a request_queue.
4678  */
4679 struct blk_mq_qe_pair {
4680 	struct list_head node;
4681 	struct request_queue *q;
4682 	struct elevator_type *type;
4683 };
4684 
4685 /*
4686  * Cache the elevator_type in qe pair list and switch the
4687  * io scheduler to 'none'
4688  */
blk_mq_elv_switch_none(struct list_head * head,struct request_queue * q)4689 static bool blk_mq_elv_switch_none(struct list_head *head,
4690 		struct request_queue *q)
4691 {
4692 	struct blk_mq_qe_pair *qe;
4693 
4694 	qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
4695 	if (!qe)
4696 		return false;
4697 
4698 	/* q->elevator needs protection from ->sysfs_lock */
4699 	mutex_lock(&q->sysfs_lock);
4700 
4701 	/* the check has to be done with holding sysfs_lock */
4702 	if (!q->elevator) {
4703 		kfree(qe);
4704 		goto unlock;
4705 	}
4706 
4707 	INIT_LIST_HEAD(&qe->node);
4708 	qe->q = q;
4709 	qe->type = q->elevator->type;
4710 	/* keep a reference to the elevator module as we'll switch back */
4711 	__elevator_get(qe->type);
4712 	list_add(&qe->node, head);
4713 	elevator_disable(q);
4714 unlock:
4715 	mutex_unlock(&q->sysfs_lock);
4716 
4717 	return true;
4718 }
4719 
blk_lookup_qe_pair(struct list_head * head,struct request_queue * q)4720 static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head,
4721 						struct request_queue *q)
4722 {
4723 	struct blk_mq_qe_pair *qe;
4724 
4725 	list_for_each_entry(qe, head, node)
4726 		if (qe->q == q)
4727 			return qe;
4728 
4729 	return NULL;
4730 }
4731 
blk_mq_elv_switch_back(struct list_head * head,struct request_queue * q)4732 static void blk_mq_elv_switch_back(struct list_head *head,
4733 				  struct request_queue *q)
4734 {
4735 	struct blk_mq_qe_pair *qe;
4736 	struct elevator_type *t;
4737 
4738 	qe = blk_lookup_qe_pair(head, q);
4739 	if (!qe)
4740 		return;
4741 	t = qe->type;
4742 	list_del(&qe->node);
4743 	kfree(qe);
4744 
4745 	mutex_lock(&q->sysfs_lock);
4746 	elevator_switch(q, t);
4747 	/* drop the reference acquired in blk_mq_elv_switch_none */
4748 	elevator_put(t);
4749 	mutex_unlock(&q->sysfs_lock);
4750 }
4751 
__blk_mq_update_nr_hw_queues(struct blk_mq_tag_set * set,int nr_hw_queues)4752 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
4753 							int nr_hw_queues)
4754 {
4755 	struct request_queue *q;
4756 	LIST_HEAD(head);
4757 	int prev_nr_hw_queues = set->nr_hw_queues;
4758 	int i;
4759 
4760 	lockdep_assert_held(&set->tag_list_lock);
4761 
4762 	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
4763 		nr_hw_queues = nr_cpu_ids;
4764 	if (nr_hw_queues < 1)
4765 		return;
4766 	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
4767 		return;
4768 
4769 	list_for_each_entry(q, &set->tag_list, tag_set_list)
4770 		blk_mq_freeze_queue(q);
4771 	/*
4772 	 * Switch IO scheduler to 'none', cleaning up the data associated
4773 	 * with the previous scheduler. We will switch back once we are done
4774 	 * updating the new sw to hw queue mappings.
4775 	 */
4776 	list_for_each_entry(q, &set->tag_list, tag_set_list)
4777 		if (!blk_mq_elv_switch_none(&head, q))
4778 			goto switch_back;
4779 
4780 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4781 		blk_mq_debugfs_unregister_hctxs(q);
4782 		blk_mq_sysfs_unregister_hctxs(q);
4783 	}
4784 
4785 	if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0)
4786 		goto reregister;
4787 
4788 fallback:
4789 	blk_mq_update_queue_map(set);
4790 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4791 		blk_mq_realloc_hw_ctxs(set, q);
4792 		blk_mq_update_poll_flag(q);
4793 		if (q->nr_hw_queues != set->nr_hw_queues) {
4794 			int i = prev_nr_hw_queues;
4795 
4796 			pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
4797 					nr_hw_queues, prev_nr_hw_queues);
4798 			for (; i < set->nr_hw_queues; i++)
4799 				__blk_mq_free_map_and_rqs(set, i);
4800 
4801 			set->nr_hw_queues = prev_nr_hw_queues;
4802 			goto fallback;
4803 		}
4804 		blk_mq_map_swqueue(q);
4805 	}
4806 
4807 reregister:
4808 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4809 		blk_mq_sysfs_register_hctxs(q);
4810 		blk_mq_debugfs_register_hctxs(q);
4811 	}
4812 
4813 switch_back:
4814 	list_for_each_entry(q, &set->tag_list, tag_set_list)
4815 		blk_mq_elv_switch_back(&head, q);
4816 
4817 	list_for_each_entry(q, &set->tag_list, tag_set_list)
4818 		blk_mq_unfreeze_queue(q);
4819 
4820 	/* Free the excess tags when nr_hw_queues shrink. */
4821 	for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
4822 		__blk_mq_free_map_and_rqs(set, i);
4823 }
4824 
blk_mq_update_nr_hw_queues(struct blk_mq_tag_set * set,int nr_hw_queues)4825 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
4826 {
4827 	mutex_lock(&set->tag_list_lock);
4828 	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
4829 	mutex_unlock(&set->tag_list_lock);
4830 }
4831 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
4832 
blk_hctx_poll(struct request_queue * q,struct blk_mq_hw_ctx * hctx,struct io_comp_batch * iob,unsigned int flags)4833 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
4834 			 struct io_comp_batch *iob, unsigned int flags)
4835 {
4836 	long state = get_current_state();
4837 	int ret;
4838 
4839 	do {
4840 		ret = q->mq_ops->poll(hctx, iob);
4841 		if (ret > 0) {
4842 			__set_current_state(TASK_RUNNING);
4843 			return ret;
4844 		}
4845 
4846 		if (signal_pending_state(state, current))
4847 			__set_current_state(TASK_RUNNING);
4848 		if (task_is_running(current))
4849 			return 1;
4850 
4851 		if (ret < 0 || (flags & BLK_POLL_ONESHOT))
4852 			break;
4853 		cpu_relax();
4854 	} while (!need_resched());
4855 
4856 	__set_current_state(TASK_RUNNING);
4857 	return 0;
4858 }
4859 
blk_mq_poll(struct request_queue * q,blk_qc_t cookie,struct io_comp_batch * iob,unsigned int flags)4860 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
4861 		struct io_comp_batch *iob, unsigned int flags)
4862 {
4863 	struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie);
4864 
4865 	return blk_hctx_poll(q, hctx, iob, flags);
4866 }
4867 
blk_rq_poll(struct request * rq,struct io_comp_batch * iob,unsigned int poll_flags)4868 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
4869 		unsigned int poll_flags)
4870 {
4871 	struct request_queue *q = rq->q;
4872 	int ret;
4873 
4874 	if (!blk_rq_is_poll(rq))
4875 		return 0;
4876 	if (!percpu_ref_tryget(&q->q_usage_counter))
4877 		return 0;
4878 
4879 	ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
4880 	blk_queue_exit(q);
4881 
4882 	return ret;
4883 }
4884 EXPORT_SYMBOL_GPL(blk_rq_poll);
4885 
blk_mq_rq_cpu(struct request * rq)4886 unsigned int blk_mq_rq_cpu(struct request *rq)
4887 {
4888 	return rq->mq_ctx->cpu;
4889 }
4890 EXPORT_SYMBOL(blk_mq_rq_cpu);
4891 
blk_mq_cancel_work_sync(struct request_queue * q)4892 void blk_mq_cancel_work_sync(struct request_queue *q)
4893 {
4894 	struct blk_mq_hw_ctx *hctx;
4895 	unsigned long i;
4896 
4897 	cancel_delayed_work_sync(&q->requeue_work);
4898 
4899 	queue_for_each_hw_ctx(q, hctx, i)
4900 		cancel_delayed_work_sync(&hctx->run_work);
4901 }
4902 
blk_mq_init(void)4903 static int __init blk_mq_init(void)
4904 {
4905 	int i;
4906 
4907 	for_each_possible_cpu(i)
4908 		init_llist_head(&per_cpu(blk_cpu_done, i));
4909 	for_each_possible_cpu(i)
4910 		INIT_CSD(&per_cpu(blk_cpu_csd, i),
4911 			 __blk_mq_complete_request_remote, NULL);
4912 	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
4913 
4914 	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
4915 				  "block/softirq:dead", NULL,
4916 				  blk_softirq_cpu_dead);
4917 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
4918 				blk_mq_hctx_notify_dead);
4919 	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
4920 				blk_mq_hctx_notify_online,
4921 				blk_mq_hctx_notify_offline);
4922 	return 0;
4923 }
4924 subsys_initcall(blk_mq_init);
4925