59 void set_capacity(struct gendisk *disk, sector_t sectors)  in set_capacity()  argument
61 	bdev_set_nr_sectors(disk->part0, sectors);  in set_capacity()
69 bool set_capacity_and_notify(struct gendisk *disk, sector_t size)  in set_capacity_and_notify()  argument
71 	sector_t capacity = get_capacity(disk);  in set_capacity_and_notify()
74 	set_capacity(disk, size);  in set_capacity_and_notify()
82 	    !disk_live(disk) ||  in set_capacity_and_notify()
83 	    (disk->flags & GENHD_FL_HIDDEN))  in set_capacity_and_notify()
87 		disk->disk_name, capacity, size);  in set_capacity_and_notify()
95 	kobject_uevent_env(&disk_to_dev(disk)->kobj, KOBJ_CHANGE, envp);  in set_capacity_and_notify()
322 void disk_uevent(struct gendisk *disk, enum kobject_action action)  in disk_uevent()  argument
328 	xa_for_each(&disk->part_tbl, idx, part) {  in disk_uevent()
343 int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode)  in disk_scan_partitions()  argument
348 	if (disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN))  in disk_scan_partitions()
350 	if (test_bit(GD_SUPPRESS_PART_SCAN, &disk->state))  in disk_scan_partitions()
352 	if (disk->open_partitions)  in disk_scan_partitions()
362 		ret = bd_prepare_to_claim(disk->part0, disk_scan_partitions,  in disk_scan_partitions()
368 	set_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
369 	bdev = blkdev_get_by_dev(disk_devt(disk), mode & ~BLK_OPEN_EXCL, NULL,  in disk_scan_partitions()
381 	clear_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
383 		bd_abort_claiming(disk->part0, disk_scan_partitions);  in disk_scan_partitions()
396 int __must_check device_add_disk(struct device *parent, struct gendisk *disk,  in device_add_disk()  argument
400 	struct device *ddev = disk_to_dev(disk);  in device_add_disk()
404 	if (queue_is_mq(disk->queue) && disk->fops->poll_bio)  in device_add_disk()
413 	elevator_init_mq(disk->queue);  in device_add_disk()
416 	disk->part0->bd_has_submit_bio = disk->fops->submit_bio != NULL;  in device_add_disk()
426 	if (disk->major) {  in device_add_disk()
427 		if (WARN_ON(!disk->minors))  in device_add_disk()
430 		if (disk->minors > DISK_MAX_PARTS) {  in device_add_disk()
433 			disk->minors = DISK_MAX_PARTS;  in device_add_disk()
435 		if (disk->first_minor > MINORMASK ||  in device_add_disk()
436 		    disk->minors > MINORMASK + 1 ||  in device_add_disk()
437 		    disk->first_minor + disk->minors > MINORMASK + 1)  in device_add_disk()
440 		if (WARN_ON(disk->minors))  in device_add_disk()
446 		disk->major = BLOCK_EXT_MAJOR;  in device_add_disk()
447 		disk->first_minor = ret;  in device_add_disk()
455 	dev_set_name(ddev, "%s", disk->disk_name);  in device_add_disk()
456 	if (!(disk->flags & GENHD_FL_HIDDEN))  in device_add_disk()
457 		ddev->devt = MKDEV(disk->major, disk->first_minor);  in device_add_disk()
462 	ret = disk_alloc_events(disk);  in device_add_disk()
478 	disk->part0->bd_holder_dir =  in device_add_disk()
480 	if (!disk->part0->bd_holder_dir) {  in device_add_disk()
484 	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);  in device_add_disk()
485 	if (!disk->slave_dir) {  in device_add_disk()
490 	ret = blk_register_queue(disk);  in device_add_disk()
494 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in device_add_disk()
495 		ret = bdi_register(disk->bdi, "%u:%u",  in device_add_disk()
496 				   disk->major, disk->first_minor);  in device_add_disk()
499 		bdi_set_owner(disk->bdi, ddev);  in device_add_disk()
501 					&disk->bdi->dev->kobj, "bdi");  in device_add_disk()
506 		if (get_capacity(disk) && !(disk->flags & GENHD_FL_NO_PART) &&  in device_add_disk()
507 		    !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state))  in device_add_disk()
508 			set_bit(GD_NEED_PART_SCAN, &disk->state);  in device_add_disk()
510 		bdev_add(disk->part0, ddev->devt);  in device_add_disk()
511 		if (get_capacity(disk))  in device_add_disk()
512 			disk_scan_partitions(disk, BLK_OPEN_READ);  in device_add_disk()
519 		disk_uevent(disk, KOBJ_ADD);  in device_add_disk()
526 		disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);  in device_add_disk()
529 	disk_update_readahead(disk);  in device_add_disk()
530 	disk_add_events(disk);  in device_add_disk()
531 	set_bit(GD_ADDED, &disk->state);  in device_add_disk()
535 	if (!(disk->flags & GENHD_FL_HIDDEN))  in device_add_disk()
536 		bdi_unregister(disk->bdi);  in device_add_disk()
538 	blk_unregister_queue(disk);  in device_add_disk()
539 	rq_qos_exit(disk->queue);  in device_add_disk()
541 	kobject_put(disk->slave_dir);  in device_add_disk()
542 	disk->slave_dir = NULL;  in device_add_disk()
544 	kobject_put(disk->part0->bd_holder_dir);  in device_add_disk()
551 	if (disk->major == BLOCK_EXT_MAJOR)  in device_add_disk()
552 		blk_free_ext_minor(disk->first_minor);  in device_add_disk()
554 	if (disk->queue->elevator)  in device_add_disk()
555 		elevator_exit(disk->queue);  in device_add_disk()
560 static void blk_report_disk_dead(struct gendisk *disk, bool surprise)  in blk_report_disk_dead()  argument
566 	xa_for_each(&disk->part_tbl, idx, bdev) {  in blk_report_disk_dead()
579 static void __blk_mark_disk_dead(struct gendisk *disk)  in __blk_mark_disk_dead()  argument
584 	if (test_and_set_bit(GD_DEAD, &disk->state))  in __blk_mark_disk_dead()
587 	if (test_bit(GD_OWNS_QUEUE, &disk->state))  in __blk_mark_disk_dead()
588 		blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue);  in __blk_mark_disk_dead()
593 	set_capacity(disk, 0);  in __blk_mark_disk_dead()
598 	blk_queue_start_drain(disk->queue);  in __blk_mark_disk_dead()
608 void blk_mark_disk_dead(struct gendisk *disk)  in blk_mark_disk_dead()  argument
610 	__blk_mark_disk_dead(disk);  in blk_mark_disk_dead()
611 	blk_report_disk_dead(disk, true);  in blk_mark_disk_dead()
634 void del_gendisk(struct gendisk *disk)  in del_gendisk()  argument
636 	struct request_queue *q = disk->queue;  in del_gendisk()
642 	if (WARN_ON_ONCE(!disk_live(disk) && !(disk->flags & GENHD_FL_HIDDEN)))  in del_gendisk()
645 	disk_del_events(disk);  in del_gendisk()
650 	mutex_lock(&disk->open_mutex);  in del_gendisk()
651 	xa_for_each(&disk->part_tbl, idx, part)  in del_gendisk()
653 	mutex_unlock(&disk->open_mutex);  in del_gendisk()
659 	if (!test_bit(GD_DEAD, &disk->state))  in del_gendisk()
660 		blk_report_disk_dead(disk, false);  in del_gendisk()
661 	__blk_mark_disk_dead(disk);  in del_gendisk()
666 	mutex_lock(&disk->open_mutex);  in del_gendisk()
667 	xa_for_each_start(&disk->part_tbl, idx, part, 1)  in del_gendisk()
669 	mutex_unlock(&disk->open_mutex);  in del_gendisk()
671 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in del_gendisk()
672 		sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");  in del_gendisk()
678 		bdi_unregister(disk->bdi);  in del_gendisk()
681 	blk_unregister_queue(disk);  in del_gendisk()
683 	kobject_put(disk->part0->bd_holder_dir);  in del_gendisk()
684 	kobject_put(disk->slave_dir);  in del_gendisk()
685 	disk->slave_dir = NULL;  in del_gendisk()
687 	part_stat_set_all(disk->part0, 0);  in del_gendisk()
688 	disk->part0->bd_stamp = 0;  in del_gendisk()
689 	sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk)));  in del_gendisk()
690 	pm_runtime_set_memalloc_noio(disk_to_dev(disk), false);  in del_gendisk()
691 	device_del(disk_to_dev(disk));  in del_gendisk()
695 	blk_throtl_cancel_bios(disk);  in del_gendisk()
716 	if (!test_bit(GD_OWNS_QUEUE, &disk->state)) {  in del_gendisk()
736 void invalidate_disk(struct gendisk *disk)  in invalidate_disk()  argument
738 	struct block_device *bdev = disk->part0;  in invalidate_disk()
742 	set_capacity(disk, 0);  in invalidate_disk()
751 	struct gendisk *disk = dev_to_disk(dev);  in disk_badblocks_show()  local
753 	if (!disk->bb)  in disk_badblocks_show()
756 	return badblocks_show(disk->bb, page, 0);  in disk_badblocks_show()
763 	struct gendisk *disk = dev_to_disk(dev);  in disk_badblocks_store()  local
765 	if (!disk->bb)  in disk_badblocks_store()
768 	return badblocks_store(disk->bb, page, len, 0);  in disk_badblocks_store()
900 	struct gendisk *disk = dev_to_disk(dev);  in disk_range_show()  local
902 	return sprintf(buf, "%d\n", disk->minors);  in disk_range_show()
908 	struct gendisk *disk = dev_to_disk(dev);  in disk_ext_range_show()  local
911 		(disk->flags & GENHD_FL_NO_PART) ? 1 : DISK_MAX_PARTS);  in disk_ext_range_show()
917 	struct gendisk *disk = dev_to_disk(dev);  in disk_removable_show()  local
920 		       (disk->flags & GENHD_FL_REMOVABLE ? 1 : 0));  in disk_removable_show()
926 	struct gendisk *disk = dev_to_disk(dev);  in disk_hidden_show()  local
929 		       (disk->flags & GENHD_FL_HIDDEN ? 1 : 0));  in disk_hidden_show()
935 	struct gendisk *disk = dev_to_disk(dev);  in disk_ro_show()  local
937 	return sprintf(buf, "%d\n", get_disk_ro(disk) ? 1 : 0);  in disk_ro_show()
1021 	struct gendisk *disk = dev_to_disk(dev);  in disk_alignment_offset_show()  local
1023 	return sprintf(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_alignment_offset_show()
1030 	struct gendisk *disk = dev_to_disk(dev);  in disk_discard_alignment_show()  local
1032 	return sprintf(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_discard_alignment_show()
1038 	struct gendisk *disk = dev_to_disk(dev);  in diskseq_show()  local
1040 	return sprintf(buf, "%llu\n", disk->diskseq);  in diskseq_show()
1114 	struct gendisk *disk = dev_to_disk(dev);  in disk_visible()  local
1116 	if (a == &dev_attr_badblocks.attr && !disk->bb)  in disk_visible()
1153 	struct gendisk *disk = dev_to_disk(dev);  in disk_release()  local
1156 	WARN_ON_ONCE(disk_live(disk));  in disk_release()
1158 	blk_trace_remove(disk->queue);  in disk_release()
1167 	if (queue_is_mq(disk->queue) &&  in disk_release()
1168 	    test_bit(GD_OWNS_QUEUE, &disk->state) &&  in disk_release()
1169 	    !test_bit(GD_ADDED, &disk->state))  in disk_release()
1170 		blk_mq_exit_queue(disk->queue);  in disk_release()
1172 	blkcg_exit_disk(disk);  in disk_release()
1174 	bioset_exit(&disk->bio_split);  in disk_release()
1176 	disk_release_events(disk);  in disk_release()
1177 	kfree(disk->random);  in disk_release()
1178 	disk_free_zone_bitmaps(disk);  in disk_release()
1179 	xa_destroy(&disk->part_tbl);  in disk_release()
1181 	disk->queue->disk = NULL;  in disk_release()
1182 	blk_put_queue(disk->queue);  in disk_release()
1184 	if (test_bit(GD_ADDED, &disk->state) && disk->fops->free_disk)  in disk_release()
1185 		disk->fops->free_disk(disk);  in disk_release()
1187 	iput(disk->part0->bd_inode);	/* frees the disk */  in disk_release()
1192 	const struct gendisk *disk = dev_to_disk(dev);  in block_uevent()  local
1194 	return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);  in block_uevent()
1205 	struct gendisk *disk = dev_to_disk(dev);  in block_devnode()  local
1207 	if (disk->fops->devnode)  in block_devnode()
1208 		return disk->fops->devnode(disk, mode);  in block_devnode()
1314 dev_t part_devt(struct gendisk *disk, u8 partno)  in part_devt()  argument
1320 	part = xa_load(&disk->part_tbl, partno);  in part_devt()
1331 	struct gendisk *disk;  in __alloc_disk_node()  local
1333 	disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);  in __alloc_disk_node()
1334 	if (!disk)  in __alloc_disk_node()
1337 	if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))  in __alloc_disk_node()
1340 	disk->bdi = bdi_alloc(node_id);  in __alloc_disk_node()
1341 	if (!disk->bdi)  in __alloc_disk_node()
1345 	disk->queue = q;  in __alloc_disk_node()
1347 	disk->part0 = bdev_alloc(disk, 0);  in __alloc_disk_node()
1348 	if (!disk->part0)  in __alloc_disk_node()
1351 	disk->node_id = node_id;  in __alloc_disk_node()
1352 	mutex_init(&disk->open_mutex);  in __alloc_disk_node()
1353 	xa_init(&disk->part_tbl);  in __alloc_disk_node()
1354 	if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))  in __alloc_disk_node()
1357 	if (blkcg_init_disk(disk))  in __alloc_disk_node()
1360 	rand_initialize_disk(disk);  in __alloc_disk_node()
1361 	disk_to_dev(disk)->class = &block_class;  in __alloc_disk_node()
1362 	disk_to_dev(disk)->type = &disk_type;  in __alloc_disk_node()
1363 	device_initialize(disk_to_dev(disk));  in __alloc_disk_node()
1364 	inc_diskseq(disk);  in __alloc_disk_node()
1365 	q->disk = disk;  in __alloc_disk_node()
1366 	lockdep_init_map(&disk->lockdep_map, "(bio completion)", lkclass, 0);  in __alloc_disk_node()
1368 	INIT_LIST_HEAD(&disk->slave_bdevs);  in __alloc_disk_node()
1370 	return disk;  in __alloc_disk_node()
1373 	xa_erase(&disk->part_tbl, 0);  in __alloc_disk_node()
1375 	xa_destroy(&disk->part_tbl);  in __alloc_disk_node()
1376 	disk->part0->bd_disk = NULL;  in __alloc_disk_node()
1377 	iput(disk->part0->bd_inode);  in __alloc_disk_node()
1379 	bdi_put(disk->bdi);  in __alloc_disk_node()
1381 	bioset_exit(&disk->bio_split);  in __alloc_disk_node()
1383 	kfree(disk);  in __alloc_disk_node()
1390 	struct gendisk *disk;  in __blk_alloc_disk()  local
1396 	disk = __alloc_disk_node(q, node, lkclass);  in __blk_alloc_disk()
1397 	if (!disk) {  in __blk_alloc_disk()
1401 	set_bit(GD_OWNS_QUEUE, &disk->state);  in __blk_alloc_disk()
1402 	return disk;  in __blk_alloc_disk()
1419 void put_disk(struct gendisk *disk)  in put_disk()  argument
1421 	if (disk)  in put_disk()
1422 		put_device(disk_to_dev(disk));  in put_disk()
1445 void set_disk_ro(struct gendisk *disk, bool read_only)  in set_disk_ro()  argument
1448 		if (test_and_set_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1451 		if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1454 	set_disk_ro_uevent(disk, read_only);  in set_disk_ro()
1458 void inc_diskseq(struct gendisk *disk)  in inc_diskseq()  argument
1460 	disk->diskseq = atomic64_inc_return(&diskseq);  in inc_diskseq()