138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()
139 						 struct bfq_group *curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_remove()
258 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_merged()
259 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
261 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()
262 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()
274 static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)  in pd_to_bfqg()
276 	return pd ? container_of(pd, struct bfq_group, pd) : NULL;  in pd_to_bfqg()
279 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()
284 static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)  in blkg_to_bfqg()
296 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()
303 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
307 	return group_entity ? container_of(group_entity, struct bfq_group,  in bfqq_group()
317 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()
322 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()
330 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()
338 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()
347 	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);  in bfqg_stats_update_legacy_io()
401 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()
403 	struct bfq_group *parent;  in bfqg_stats_xfer_dead()
419 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
521 	struct bfq_group *bfqg;  in bfq_pd_alloc()
540 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()
561 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()
569 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()
574 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()
575 					struct bfq_group *parent)  in bfq_group_set_parent()
584 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()
586 	struct bfq_group *parent;  in bfq_link_bfqg()
596 		struct bfq_group *curr_bfqg = container_of(entity,  in bfq_link_bfqg()
597 						struct bfq_group, entity);  in bfq_link_bfqg()
607 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
610 	struct bfq_group *bfqg;  in bfq_bio_bfqg()
644 		   struct bfq_group *bfqg)  in bfq_bfqq_move()
647 	struct bfq_group *old_parent = bfqq_group(bfqq);  in bfq_bfqq_move()
719 				     struct bfq_group *bfqg)  in __bfq_bic_change_cgroup()
776 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()
898 				       struct bfq_group *bfqg,  in bfq_reparent_active_queues()
925 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()
992 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()
1016 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()
1034 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()
1083 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()
1100 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()
1219 	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);  in bfqg_prfill_sectors()
1238 			offsetof(struct bfq_group, stats.bytes), &tmp);  in bfqg_prfill_sectors_recursive()
1255 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()
1277 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1321 		.private = offsetof(struct bfq_group, stats.bytes),
1326 		.private = offsetof(struct bfq_group, stats.ios),
1332 		.private = offsetof(struct bfq_group, stats.time),
1341 		.private = offsetof(struct bfq_group, stats.service_time),
1346 		.private = offsetof(struct bfq_group, stats.wait_time),
1351 		.private = offsetof(struct bfq_group, stats.merged),
1356 		.private = offsetof(struct bfq_group, stats.queued),
1364 		.private = offsetof(struct bfq_group, stats.bytes),
1369 		.private = offsetof(struct bfq_group, stats.ios),
1375 		.private = offsetof(struct bfq_group, stats.time),
1384 		.private = offsetof(struct bfq_group, stats.service_time),
1389 		.private = offsetof(struct bfq_group, stats.wait_time),
1394 		.private = offsetof(struct bfq_group, stats.merged),
1399 		.private = offsetof(struct bfq_group, stats.queued),
1408 		.private = offsetof(struct bfq_group, stats.group_wait_time),
1413 		.private = offsetof(struct bfq_group, stats.idle_time),
1418 		.private = offsetof(struct bfq_group, stats.empty_time),
1423 		.private = offsetof(struct bfq_group, stats.dequeue),
1443 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()
1445 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
1465 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
1470 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
1475 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}  in bfqg_and_blkg_put()
1477 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1479 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()