138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()  argument
141 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_group_wait_time()
145 	if (bfqg == curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()  argument
168 	bfq_stat_add(&bfqg->stats.dequeue, 1);  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()  argument
173 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()  argument
192 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()  argument
206 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()  argument
214 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
225 	blkg_rwstat_add(&bfqg->stats.queued, opf, 1);  in bfqg_stats_update_io_add()
226 	bfqg_stats_end_empty_time(&bfqg->stats);  in bfqg_stats_update_io_add()
227 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))  in bfqg_stats_update_io_add()
228 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_remove()  argument
233 	blkg_rwstat_add(&bfqg->stats.queued, opf, -1);  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_merged()  argument
238 	blkg_rwstat_add(&bfqg->stats.merged, opf, 1);  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
244 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_remove()  argument
258 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_merged()  argument
259 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
261 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()  argument
262 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()  argument
279 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()  argument
281 	return pd_to_blkg(&bfqg->pd);  in bfqg_to_blkg()
296 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()  argument
298 	struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;  in bfqg_parent()
317 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()  argument
319 	bfqg->ref++;  in bfqg_get()
322 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()  argument
324 	bfqg->ref--;  in bfqg_put()
326 	if (bfqg->ref == 0)  in bfqg_put()
327 		kfree(bfqg);  in bfqg_put()
330 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()  argument
333 	bfqg_get(bfqg);  in bfqg_and_blkg_get()
335 	blkg_get(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_get()
338 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()  argument
340 	blkg_put(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_put()
342 	bfqg_put(bfqg);  in bfqg_and_blkg_put()
347 	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);  in bfqg_stats_update_legacy_io()  local
349 	if (!bfqg)  in bfqg_stats_update_legacy_io()
352 	blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));  in bfqg_stats_update_legacy_io()
353 	blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);  in bfqg_stats_update_legacy_io()
401 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()  argument
405 	if (!bfqg) /* root_group */  in bfqg_stats_xfer_dead()
408 	parent = bfqg_parent(bfqg);  in bfqg_stats_xfer_dead()
410 	lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);  in bfqg_stats_xfer_dead()
415 	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);  in bfqg_stats_xfer_dead()
416 	bfqg_stats_reset(&bfqg->stats);  in bfqg_stats_xfer_dead()
419 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
432 		bfqg_and_blkg_get(bfqg);  in bfq_init_entity()
434 	entity->parent = bfqg->my_entity; /* NULL for root group */  in bfq_init_entity()
435 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
521 	struct bfq_group *bfqg;  in bfq_pd_alloc()  local
523 	bfqg = kzalloc_node(sizeof(*bfqg), gfp, q->node);  in bfq_pd_alloc()
524 	if (!bfqg)  in bfq_pd_alloc()
527 	if (bfqg_stats_init(&bfqg->stats, gfp)) {  in bfq_pd_alloc()
528 		kfree(bfqg);  in bfq_pd_alloc()
533 	bfqg_get(bfqg);  in bfq_pd_alloc()
534 	return &bfqg->pd;  in bfq_pd_alloc()
540 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()  local
542 	struct bfq_entity *entity = &bfqg->entity;  in bfq_pd_init()
546 	entity->my_sched_data = &bfqg->sched_data;  in bfq_pd_init()
549 	bfqg->my_entity = entity; /*  in bfq_pd_init()
553 	bfqg->bfqd = bfqd;  in bfq_pd_init()
554 	bfqg->active_entities = 0;  in bfq_pd_init()
555 	bfqg->online = true;  in bfq_pd_init()
556 	bfqg->rq_pos_tree = RB_ROOT;  in bfq_pd_init()
561 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()  local
563 	bfqg_stats_exit(&bfqg->stats);  in bfq_pd_free()
564 	bfqg_put(bfqg);  in bfq_pd_free()
569 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()  local
571 	bfqg_stats_reset(&bfqg->stats);  in bfq_pd_reset_stats()
574 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()  argument
579 	entity = &bfqg->entity;  in bfq_group_set_parent()
584 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()  argument
594 	entity = &bfqg->entity;  in bfq_link_bfqg()
610 	struct bfq_group *bfqg;  in bfq_bio_bfqg()  local
617 		bfqg = blkg_to_bfqg(blkg);  in bfq_bio_bfqg()
618 		if (bfqg->online) {  in bfq_bio_bfqg()
620 			return bfqg;  in bfq_bio_bfqg()
644 		   struct bfq_group *bfqg)  in bfq_bfqq_move()  argument
653 	if (old_parent == bfqg)  in bfq_bfqq_move()
690 	entity->parent = bfqg->my_entity;  in bfq_bfqq_move()
691 	entity->sched_data = &bfqg->sched_data;  in bfq_bfqq_move()
693 	bfqg_and_blkg_get(bfqg);  in bfq_bfqq_move()
719 				     struct bfq_group *bfqg)  in __bfq_bic_change_cgroup()  argument
728 		if (entity->sched_data != &bfqg->sched_data) {  in __bfq_bic_change_cgroup()
737 			if (sync_bfqq->entity.sched_data != &bfqg->sched_data)  in __bfq_bic_change_cgroup()
738 				bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in __bfq_bic_change_cgroup()
749 				    &bfqg->sched_data)  in __bfq_bic_change_cgroup()
770 	return bfqg;  in __bfq_bic_change_cgroup()
776 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()  local
779 	serial_nr = bfqg_to_blkg(bfqg)->blkcg->css.serial_nr;  in bfq_bic_update_cgroup()
792 	bfq_link_bfqg(bfqd, bfqg);  in bfq_bic_update_cgroup()
793 	__bfq_bic_change_cgroup(bfqd, bic, bfqg);  in bfq_bic_update_cgroup()
844 	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));  in bfq_bic_update_cgroup()
898 				       struct bfq_group *bfqg,  in bfq_reparent_active_queues()  argument
908 	if (bfqg->sched_data.in_service_entity)  in bfq_reparent_active_queues()
910 					 bfqg->sched_data.in_service_entity,  in bfq_reparent_active_queues()
925 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()  local
926 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()
927 	struct bfq_entity *entity = bfqg->my_entity;  in bfq_pd_offline()
941 		st = bfqg->sched_data.service_tree + i;  in bfq_pd_offline()
955 		bfq_reparent_active_queues(bfqd, bfqg, st, i);  in bfq_pd_offline()
974 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
975 	bfqg->online = false;  in bfq_pd_offline()
984 	bfqg_stats_xfer_dead(bfqg);  in bfq_pd_offline()
992 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()  local
994 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
1016 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()  local
1018 	if (!bfqg->entity.dev_weight)  in bfqg_prfill_weight_device()
1020 	return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);  in bfqg_prfill_weight_device()
1034 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()  argument
1038 	bfqg->entity.dev_weight = dev_weight;  in bfq_group_set_weight()
1045 	if ((unsigned short)weight != bfqg->entity.new_weight) {  in bfq_group_set_weight()
1046 		bfqg->entity.new_weight = (unsigned short)weight;  in bfq_group_set_weight()
1063 		bfqg->entity.prio_changed = 1;  in bfq_group_set_weight()
1083 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()  local
1085 		if (bfqg)  in bfq_io_set_weight_legacy()
1086 			bfq_group_set_weight(bfqg, val, 0);  in bfq_io_set_weight_legacy()
1100 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()  local
1119 	bfqg = blkg_to_bfqg(ctx.blkg);  in bfq_io_set_device_weight()
1123 		bfq_group_set_weight(bfqg, bfqg->entity.weight, v);  in bfq_io_set_device_weight()
1219 	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);  in bfqg_prfill_sectors()  local
1220 	u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);  in bfqg_prfill_sectors()
1255 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()  local
1256 	u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);  in bfqg_prfill_avg_queue_size()
1260 		v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);  in bfqg_prfill_avg_queue_size()
1443 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()  argument
1445 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
1455 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
1475 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}  in bfqg_and_blkg_put()  argument
1479 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()  local
1482 	bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);  in bfq_create_group_hierarchy()
1483 	if (!bfqg)  in bfq_create_group_hierarchy()
1487 		bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;  in bfq_create_group_hierarchy()
1489 	return bfqg;  in bfq_create_group_hierarchy()