64 	struct scrub_block	*sblock;  member
210 				struct scrub_block *sblock,
212 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
218 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
219 static int scrub_write_sector_to_dev_replace(struct scrub_block *sblock,
221 static int scrub_checksum_data(struct scrub_block *sblock);
222 static int scrub_checksum_tree_block(struct scrub_block *sblock);
223 static int scrub_checksum_super(struct scrub_block *sblock);
224 static void scrub_block_put(struct scrub_block *sblock);
235 static void scrub_block_complete(struct scrub_block *sblock);
540 			scrub_block_put(sbio->sectors[i]->sblock);  in scrub_free_ctx()
713 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
730 	WARN_ON(sblock->sector_count < 1);  in scrub_print_warning()
731 	dev = sblock->sectors[0]->dev;  in scrub_print_warning()
732 	fs_info = sblock->sctx->fs_info;  in scrub_print_warning()
738 	swarn.physical = sblock->sectors[0]->physical;  in scrub_print_warning()
739 	swarn.logical = sblock->sectors[0]->logical;  in scrub_print_warning()
1184 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1189 			for (i = 0; i < sblock->sector_count; i++) {  in scrub_handle_errored_block()
1190 				sblock->sectors[i]->sblock = NULL;  in scrub_handle_errored_block()
1191 				recover = sblock->sectors[i]->recover;  in scrub_handle_errored_block()
1194 					sblock->sectors[i]->recover = NULL;  in scrub_handle_errored_block()
1196 				scrub_sector_put(sblock->sectors[i]);  in scrub_handle_errored_block()
1310 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1313 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1314 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1326 			sblock->sectors[sector_index] = sector;  in scrub_setup_recheck_block()
1327 			sector->sblock = sblock;  in scrub_setup_recheck_block()
1356 			sblock->sector_count++;  in scrub_setup_recheck_block()
1390 	mirror_num = sector->sblock->sectors[0]->mirror_num;  in scrub_submit_raid56_bio_wait()
1402 					  struct scrub_block *sblock)  in scrub_recheck_block_on_raid56()  argument
1404 	struct scrub_sector *first_sector = sblock->sectors[0];  in scrub_recheck_block_on_raid56()
1415 	for (i = 0; i < sblock->sector_count; i++) {  in scrub_recheck_block_on_raid56()
1416 		struct scrub_sector *sector = sblock->sectors[i];  in scrub_recheck_block_on_raid56()
1429 	scrub_recheck_block_checksum(sblock);  in scrub_recheck_block_on_raid56()
1433 	for (i = 0; i < sblock->sector_count; i++)  in scrub_recheck_block_on_raid56()
1434 		sblock->sectors[i]->io_error = 1;  in scrub_recheck_block_on_raid56()
1436 	sblock->no_io_error_seen = 0;  in scrub_recheck_block_on_raid56()
1447 				struct scrub_block *sblock,  in scrub_recheck_block()  argument
1452 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1455 	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->sectors[0]))  in scrub_recheck_block()
1456 		return scrub_recheck_block_on_raid56(fs_info, sblock);  in scrub_recheck_block()
1458 	for (i = 0; i < sblock->sector_count; i++) {  in scrub_recheck_block()
1459 		struct scrub_sector *sector = sblock->sectors[i];  in scrub_recheck_block()
1465 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1477 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1483 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1484 		scrub_recheck_block_checksum(sblock);  in scrub_recheck_block()
1496 static void scrub_recheck_block_checksum(struct scrub_block *sblock)  in scrub_recheck_block_checksum()  argument
1498 	sblock->header_error = 0;  in scrub_recheck_block_checksum()
1499 	sblock->checksum_error = 0;  in scrub_recheck_block_checksum()
1500 	sblock->generation_error = 0;  in scrub_recheck_block_checksum()
1502 	if (sblock->sectors[0]->flags & BTRFS_EXTENT_FLAG_DATA)  in scrub_recheck_block_checksum()
1503 		scrub_checksum_data(sblock);  in scrub_recheck_block_checksum()
1505 		scrub_checksum_tree_block(sblock);  in scrub_recheck_block_checksum()
1568 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1570 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_write_block_to_dev_replace()
1577 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1580 	for (i = 0; i < sblock->sector_count; i++) {  in scrub_write_block_to_dev_replace()
1583 		ret = scrub_write_sector_to_dev_replace(sblock, i);  in scrub_write_block_to_dev_replace()
1589 static int scrub_write_sector_to_dev_replace(struct scrub_block *sblock, int sector_num)  in scrub_write_sector_to_dev_replace()  argument
1591 	struct scrub_sector *sector = sblock->sectors[sector_num];  in scrub_write_sector_to_dev_replace()
1597 	return scrub_add_sector_to_wr_bio(sblock->sctx, sector);  in scrub_write_sector_to_dev_replace()
1751 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1764 	sblock->header_error = 0;  in scrub_checksum()
1765 	sblock->generation_error = 0;  in scrub_checksum()
1766 	sblock->checksum_error = 0;  in scrub_checksum()
1768 	WARN_ON(sblock->sector_count < 1);  in scrub_checksum()
1769 	flags = sblock->sectors[0]->flags;  in scrub_checksum()
1772 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1774 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1776 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1780 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1785 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1787 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1794 	BUG_ON(sblock->sector_count < 1);  in scrub_checksum_data()
1795 	sector = sblock->sectors[0];  in scrub_checksum_data()
1811 		sblock->checksum_error = 1;  in scrub_checksum_data()
1812 	return sblock->checksum_error;  in scrub_checksum_data()
1815 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1817 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1834 	BUG_ON(sblock->sector_count < 1);  in scrub_checksum_tree_block()
1837 	ASSERT(sblock->sector_count == num_sectors);  in scrub_checksum_tree_block()
1839 	sector = sblock->sectors[0];  in scrub_checksum_tree_block()
1850 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1853 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1854 		sblock->generation_error = 1;  in scrub_checksum_tree_block()
1858 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1862 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1870 		kaddr = page_address(sblock->sectors[i]->page);  in scrub_checksum_tree_block()
1876 		sblock->checksum_error = 1;  in scrub_checksum_tree_block()
1878 	return sblock->header_error || sblock->checksum_error;  in scrub_checksum_tree_block()
1881 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1884 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1893 	BUG_ON(sblock->sector_count < 1);  in scrub_checksum_super()
1894 	sector = sblock->sectors[0];  in scrub_checksum_super()
1935 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
1937 	refcount_inc(&sblock->refs);  in scrub_block_get()
1940 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
1942 	if (refcount_dec_and_test(&sblock->refs)) {  in scrub_block_put()
1945 		if (sblock->sparity)  in scrub_block_put()
1946 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
1948 		for (i = 0; i < sblock->sector_count; i++)  in scrub_block_put()
1949 			scrub_sector_put(sblock->sectors[i]);  in scrub_block_put()
1950 		kfree(sblock);  in scrub_block_put()
2046 	struct scrub_block *sblock = sector->sblock;  in scrub_add_sector_to_rd_bio()  local
2102 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_sector_to_rd_bio()
2103 	atomic_inc(&sblock->outstanding_sectors);  in scrub_add_sector_to_rd_bio()
2113 	struct scrub_block *sblock = bio->bi_private;  in scrub_missing_raid56_end_io()  local
2114 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_missing_raid56_end_io()
2117 		sblock->no_io_error_seen = 0;  in scrub_missing_raid56_end_io()
2121 	queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2126 	struct scrub_block *sblock = container_of(work, struct scrub_block, work);  in scrub_missing_raid56_worker()  local
2127 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_worker()
2132 	logical = sblock->sectors[0]->logical;  in scrub_missing_raid56_worker()
2133 	dev = sblock->sectors[0]->dev;  in scrub_missing_raid56_worker()
2135 	if (sblock->no_io_error_seen)  in scrub_missing_raid56_worker()
2136 		scrub_recheck_block_checksum(sblock);  in scrub_missing_raid56_worker()
2138 	if (!sblock->no_io_error_seen) {  in scrub_missing_raid56_worker()
2145 	} else if (sblock->header_error || sblock->checksum_error) {  in scrub_missing_raid56_worker()
2153 		scrub_write_block_to_dev_replace(sblock);  in scrub_missing_raid56_worker()
2162 	scrub_block_put(sblock);  in scrub_missing_raid56_worker()
2166 static void scrub_missing_raid56_pages(struct scrub_block *sblock)  in scrub_missing_raid56_pages()  argument
2168 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_pages()
2170 	u64 length = sblock->sector_count << fs_info->sectorsize_bits;  in scrub_missing_raid56_pages()
2171 	u64 logical = sblock->sectors[0]->logical;  in scrub_missing_raid56_pages()
2197 	bio->bi_private = sblock;  in scrub_missing_raid56_pages()
2204 	for (i = 0; i < sblock->sector_count; i++) {  in scrub_missing_raid56_pages()
2205 		struct scrub_sector *sector = sblock->sectors[i];  in scrub_missing_raid56_pages()
2214 	INIT_WORK(&sblock->work, scrub_missing_raid56_worker);  in scrub_missing_raid56_pages()
2215 	scrub_block_get(sblock);  in scrub_missing_raid56_pages()
2235 	struct scrub_block *sblock;  in scrub_sectors()  local
2239 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_sectors()
2240 	if (!sblock) {  in scrub_sectors()
2249 	refcount_set(&sblock->refs, 1);  in scrub_sectors()
2250 	sblock->sctx = sctx;  in scrub_sectors()
2251 	sblock->no_io_error_seen = 1;  in scrub_sectors()
2268 			scrub_block_put(sblock);  in scrub_sectors()
2273 		sblock->sectors[index] = sector;  in scrub_sectors()
2274 		sector->sblock = sblock;  in scrub_sectors()
2288 		sblock->sector_count++;  in scrub_sectors()
2298 	WARN_ON(sblock->sector_count == 0);  in scrub_sectors()
2304 		scrub_missing_raid56_pages(sblock);  in scrub_sectors()
2306 		for (index = 0; index < sblock->sector_count; index++) {  in scrub_sectors()
2307 			struct scrub_sector *sector = sblock->sectors[index];  in scrub_sectors()
2312 				scrub_block_put(sblock);  in scrub_sectors()
2322 	scrub_block_put(sblock);  in scrub_sectors()
2349 			sector->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2356 		struct scrub_block *sblock = sector->sblock;  in scrub_bio_end_io_worker()  local
2358 		if (atomic_dec_and_test(&sblock->outstanding_sectors))  in scrub_bio_end_io_worker()
2359 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2360 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2418 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2422 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2424 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2431 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2432 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2433 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2436 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2437 		u64 start = sblock->sectors[0]->logical;  in scrub_block_complete()
2438 		u64 end = sblock->sectors[sblock->sector_count - 1]->logical +  in scrub_block_complete()
2439 			  sblock->sctx->fs_info->sectorsize;  in scrub_block_complete()
2442 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2586 	struct scrub_block *sblock;  in scrub_sectors_for_parity()  local
2592 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_sectors_for_parity()
2593 	if (!sblock) {  in scrub_sectors_for_parity()
2602 	refcount_set(&sblock->refs, 1);  in scrub_sectors_for_parity()
2603 	sblock->sctx = sctx;  in scrub_sectors_for_parity()
2604 	sblock->no_io_error_seen = 1;  in scrub_sectors_for_parity()
2605 	sblock->sparity = sparity;  in scrub_sectors_for_parity()
2617 			scrub_block_put(sblock);  in scrub_sectors_for_parity()
2623 		sblock->sectors[index] = sector;  in scrub_sectors_for_parity()
2627 		sector->sblock = sblock;  in scrub_sectors_for_parity()
2640 		sblock->sector_count++;  in scrub_sectors_for_parity()
2652 	WARN_ON(sblock->sector_count == 0);  in scrub_sectors_for_parity()
2653 	for (index = 0; index < sblock->sector_count; index++) {  in scrub_sectors_for_parity()
2654 		struct scrub_sector *sector = sblock->sectors[index];  in scrub_sectors_for_parity()
2659 			scrub_block_put(sblock);  in scrub_sectors_for_parity()
2665 	scrub_block_put(sblock);  in scrub_sectors_for_parity()