95 			   struct blk_flush_queue *fq, blk_opf_t flags);
100 	return blk_mq_map_queue(q, REQ_OP_FLUSH, ctx)->fq;  in blk_get_flush_queue()
164 				   struct blk_flush_queue *fq,  in blk_flush_complete_seq()  argument
168 	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];  in blk_flush_complete_seq()
185 			fq->flush_pending_since = jiffies;  in blk_flush_complete_seq()
190 		fq->flush_data_in_flight++;  in blk_flush_complete_seq()
213 	blk_kick_flush(q, fq, cmd_flags);  in blk_flush_complete_seq()
223 	struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);  in flush_end_io()  local
226 	spin_lock_irqsave(&fq->mq_flush_lock, flags);  in flush_end_io()
229 		fq->rq_status = error;  in flush_end_io()
230 		spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in flush_end_io()
241 	if (fq->rq_status != BLK_STS_OK) {  in flush_end_io()
242 		error = fq->rq_status;  in flush_end_io()
243 		fq->rq_status = BLK_STS_OK;  in flush_end_io()
253 	running = &fq->flush_queue[fq->flush_running_idx];  in flush_end_io()
254 	BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);  in flush_end_io()
257 	fq->flush_running_idx ^= 1;  in flush_end_io()
264 		blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
267 	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in flush_end_io()
289 static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,  in blk_kick_flush()  argument
292 	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];  in blk_kick_flush()
295 	struct request *flush_rq = fq->flush_rq;  in blk_kick_flush()
298 	if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))  in blk_kick_flush()
302 	if (fq->flush_data_in_flight &&  in blk_kick_flush()
304 			fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))  in blk_kick_flush()
311 	fq->flush_pending_idx ^= 1;  in blk_kick_flush()
365 	struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);  in mq_flush_data_end_io()  local
376 	spin_lock_irqsave(&fq->mq_flush_lock, flags);  in mq_flush_data_end_io()
377 	fq->flush_data_in_flight--;  in mq_flush_data_end_io()
383 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
384 	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in mq_flush_data_end_io()
408 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()  local
452 		spin_lock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
453 		fq->flush_data_in_flight++;  in blk_insert_flush()
454 		spin_unlock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
462 		spin_lock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
463 		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()
464 		spin_unlock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
488 	struct blk_flush_queue *fq;  in blk_alloc_flush_queue()  local
491 	fq = kzalloc_node(sizeof(*fq), flags, node);  in blk_alloc_flush_queue()
492 	if (!fq)  in blk_alloc_flush_queue()
495 	spin_lock_init(&fq->mq_flush_lock);  in blk_alloc_flush_queue()
498 	fq->flush_rq = kzalloc_node(rq_sz, flags, node);  in blk_alloc_flush_queue()
499 	if (!fq->flush_rq)  in blk_alloc_flush_queue()
502 	INIT_LIST_HEAD(&fq->flush_queue[0]);  in blk_alloc_flush_queue()
503 	INIT_LIST_HEAD(&fq->flush_queue[1]);  in blk_alloc_flush_queue()
505 	return fq;  in blk_alloc_flush_queue()
508 	kfree(fq);  in blk_alloc_flush_queue()
513 void blk_free_flush_queue(struct blk_flush_queue *fq)  in blk_free_flush_queue()  argument
516 	if (!fq)  in blk_free_flush_queue()
519 	kfree(fq->flush_rq);  in blk_free_flush_queue()
520 	kfree(fq);  in blk_free_flush_queue()
544 	lockdep_set_class(&hctx->fq->mq_flush_lock, key);  in blk_mq_hctx_set_fq_lock_class()