3544 				      struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_clustered()  argument
3548 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_clustered()
3553 	cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);  in find_free_extent_clustered()
3557 	    !block_group_bits(cluster_bg, ffe_ctl->flags)))  in find_free_extent_clustered()
3561 			ffe_ctl->num_bytes, cluster_bg->start,  in find_free_extent_clustered()
3562 			&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3567 				ffe_ctl->search_start, ffe_ctl->num_bytes);  in find_free_extent_clustered()
3569 		ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3586 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {  in find_free_extent_clustered()
3588 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3596 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3599 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {  in find_free_extent_clustered()
3605 			ffe_ctl->empty_cluster + ffe_ctl->empty_size,  in find_free_extent_clustered()
3607 	ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,  in find_free_extent_clustered()
3608 			ffe_ctl->num_bytes, aligned_cluster);  in find_free_extent_clustered()
3612 				ffe_ctl->num_bytes, ffe_ctl->search_start,  in find_free_extent_clustered()
3613 				&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3618 					ffe_ctl->search_start,  in find_free_extent_clustered()
3619 					ffe_ctl->num_bytes);  in find_free_extent_clustered()
3620 			ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3623 	} else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&  in find_free_extent_clustered()
3624 		   !ffe_ctl->retry_clustered) {  in find_free_extent_clustered()
3627 		ffe_ctl->retry_clustered = true;  in find_free_extent_clustered()
3628 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_clustered()
3629 				ffe_ctl->empty_cluster + ffe_ctl->empty_size);  in find_free_extent_clustered()
3648 					struct find_free_extent_ctl *ffe_ctl)  in find_free_extent_unclustered()  argument
3650 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_unclustered()
3663 	if (ffe_ctl->cached) {  in find_free_extent_unclustered()
3669 		    ffe_ctl->num_bytes + ffe_ctl->empty_cluster +  in find_free_extent_unclustered()
3670 		    ffe_ctl->empty_size) {  in find_free_extent_unclustered()
3671 			ffe_ctl->total_free_space = max_t(u64,  in find_free_extent_unclustered()
3672 					ffe_ctl->total_free_space,  in find_free_extent_unclustered()
3680 	offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,  in find_free_extent_unclustered()
3681 			ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent_unclustered()
3682 			&ffe_ctl->max_extent_size);  in find_free_extent_unclustered()
3693 	if (!offset && !ffe_ctl->retry_unclustered && !ffe_ctl->cached &&  in find_free_extent_unclustered()
3694 	    ffe_ctl->loop > LOOP_CACHING_NOWAIT) {  in find_free_extent_unclustered()
3695 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_unclustered()
3696 						      ffe_ctl->empty_size);  in find_free_extent_unclustered()
3697 		ffe_ctl->retry_unclustered = true;  in find_free_extent_unclustered()
3702 	ffe_ctl->found_offset = offset;  in find_free_extent_unclustered()
3707 				   struct find_free_extent_ctl *ffe_ctl,  in do_allocation_clustered()  argument
3713 	if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) {  in do_allocation_clustered()
3714 		ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation_clustered()
3720 	return find_free_extent_unclustered(block_group, ffe_ctl);  in do_allocation_clustered()
3745 			       struct find_free_extent_ctl *ffe_ctl,  in do_allocation_zoned()  argument
3752 	u64 num_bytes = ffe_ctl->num_bytes;  in do_allocation_zoned()
3768 	if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) ||  in do_allocation_zoned()
3769 			   (!ffe_ctl->for_treelog && bytenr == log_bytenr)))  in do_allocation_zoned()
3782 	    ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) ||  in do_allocation_zoned()
3783 	     (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr)))  in do_allocation_zoned()
3816 	ASSERT(!ffe_ctl->for_treelog ||  in do_allocation_zoned()
3819 	ASSERT(!ffe_ctl->for_data_reloc ||  in do_allocation_zoned()
3832 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg &&  in do_allocation_zoned()
3842 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg &&  in do_allocation_zoned()
3851 		if (ffe_ctl->max_extent_size < avail) {  in do_allocation_zoned()
3856 			ffe_ctl->max_extent_size = avail;  in do_allocation_zoned()
3857 			ffe_ctl->total_free_space = avail;  in do_allocation_zoned()
3863 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg)  in do_allocation_zoned()
3866 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg)  in do_allocation_zoned()
3869 	ffe_ctl->found_offset = start + block_group->alloc_offset;  in do_allocation_zoned()
3880 	ffe_ctl->search_start = ffe_ctl->found_offset;  in do_allocation_zoned()
3883 	if (ret && ffe_ctl->for_treelog)  in do_allocation_zoned()
3885 	if (ret && ffe_ctl->for_data_reloc &&  in do_allocation_zoned()
3911 			 struct find_free_extent_ctl *ffe_ctl,  in do_allocation()  argument
3914 	switch (ffe_ctl->policy) {  in do_allocation()
3916 		return do_allocation_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation()
3918 		return do_allocation_zoned(block_group, ffe_ctl, bg_ret);  in do_allocation()
3925 				struct find_free_extent_ctl *ffe_ctl,  in release_block_group()  argument
3928 	switch (ffe_ctl->policy) {  in release_block_group()
3930 		ffe_ctl->retry_clustered = false;  in release_block_group()
3931 		ffe_ctl->retry_unclustered = false;  in release_block_group()
3941 	       ffe_ctl->index);  in release_block_group()
3945 static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl,  in found_extent_clustered()  argument
3948 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in found_extent_clustered()
3950 	if (!ffe_ctl->use_cluster && last_ptr) {  in found_extent_clustered()
3957 static void found_extent(struct find_free_extent_ctl *ffe_ctl,  in found_extent()  argument
3960 	switch (ffe_ctl->policy) {  in found_extent()
3962 		found_extent_clustered(ffe_ctl, ins);  in found_extent()
3973 				    struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk_zoned()  argument
3976 	if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))  in can_allocate_chunk_zoned()
3986 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {  in can_allocate_chunk_zoned()
4000 	if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size)  in can_allocate_chunk_zoned()
4010 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)  in can_allocate_chunk_zoned()
4022 			      struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk()  argument
4024 	switch (ffe_ctl->policy) {  in can_allocate_chunk()
4028 		return can_allocate_chunk_zoned(fs_info, ffe_ctl);  in can_allocate_chunk()
4034 static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl)  in chunk_allocation_failed()  argument
4036 	switch (ffe_ctl->policy) {  in chunk_allocation_failed()
4042 		ffe_ctl->loop = LOOP_NO_EMPTY_SIZE;  in chunk_allocation_failed()
4059 					struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_update_loop()  argument
4065 	if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&  in find_free_extent_update_loop()
4066 	    ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)  in find_free_extent_update_loop()
4067 		ffe_ctl->orig_have_caching_bg = true;  in find_free_extent_update_loop()
4070 		found_extent(ffe_ctl, ins);  in find_free_extent_update_loop()
4074 	if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg)  in find_free_extent_update_loop()
4077 	ffe_ctl->index++;  in find_free_extent_update_loop()
4078 	if (ffe_ctl->index < BTRFS_NR_RAID_TYPES)  in find_free_extent_update_loop()
4089 	if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4090 		ffe_ctl->index = 0;  in find_free_extent_update_loop()
4091 		if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) {  in find_free_extent_update_loop()
4097 			if (ffe_ctl->orig_have_caching_bg || !full_search)  in find_free_extent_update_loop()
4098 				ffe_ctl->loop = LOOP_CACHING_WAIT;  in find_free_extent_update_loop()
4100 				ffe_ctl->loop = LOOP_ALLOC_CHUNK;  in find_free_extent_update_loop()
4102 			ffe_ctl->loop++;  in find_free_extent_update_loop()
4105 		if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {  in find_free_extent_update_loop()
4110 			ret = can_allocate_chunk(fs_info, ffe_ctl);  in find_free_extent_update_loop()
4125 			ret = btrfs_chunk_alloc(trans, ffe_ctl->flags,  in find_free_extent_update_loop()
4130 				ret = chunk_allocation_failed(ffe_ctl);  in find_free_extent_update_loop()
4141 		if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4142 			if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED)  in find_free_extent_update_loop()
4149 			if (ffe_ctl->empty_size == 0 &&  in find_free_extent_update_loop()
4150 			    ffe_ctl->empty_cluster == 0)  in find_free_extent_update_loop()
4152 			ffe_ctl->empty_size = 0;  in find_free_extent_update_loop()
4153 			ffe_ctl->empty_cluster = 0;  in find_free_extent_update_loop()
4161 					struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation_clustered()  argument
4178 		    ffe_ctl->num_bytes > space_info->max_extent_size) {  in prepare_allocation_clustered()
4183 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4188 	ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info,  in prepare_allocation_clustered()
4189 					       &ffe_ctl->empty_cluster);  in prepare_allocation_clustered()
4190 	if (ffe_ctl->last_ptr) {  in prepare_allocation_clustered()
4191 		struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in prepare_allocation_clustered()
4195 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4202 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4203 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4212 			      struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation()  argument
4216 	switch (ffe_ctl->policy) {  in prepare_allocation()
4218 		return prepare_allocation_clustered(fs_info, ffe_ctl,  in prepare_allocation()
4221 		if (ffe_ctl->for_treelog) {  in prepare_allocation()
4224 				ffe_ctl->hint_byte = fs_info->treelog_bg;  in prepare_allocation()
4227 		if (ffe_ctl->for_data_reloc) {  in prepare_allocation()
4230 				ffe_ctl->hint_byte = fs_info->data_reloc_bg;  in prepare_allocation()
4266 				     struct find_free_extent_ctl *ffe_ctl)  in find_free_extent()  argument
4275 	WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize);  in find_free_extent()
4277 	ffe_ctl->search_start = 0;  in find_free_extent()
4279 	ffe_ctl->empty_cluster = 0;  in find_free_extent()
4280 	ffe_ctl->last_ptr = NULL;  in find_free_extent()
4281 	ffe_ctl->use_cluster = true;  in find_free_extent()
4282 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4283 	ffe_ctl->orig_have_caching_bg = false;  in find_free_extent()
4284 	ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags);  in find_free_extent()
4285 	ffe_ctl->loop = 0;  in find_free_extent()
4287 	ffe_ctl->retry_clustered = false;  in find_free_extent()
4288 	ffe_ctl->retry_unclustered = false;  in find_free_extent()
4289 	ffe_ctl->cached = 0;  in find_free_extent()
4290 	ffe_ctl->max_extent_size = 0;  in find_free_extent()
4291 	ffe_ctl->total_free_space = 0;  in find_free_extent()
4292 	ffe_ctl->found_offset = 0;  in find_free_extent()
4293 	ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED;  in find_free_extent()
4296 		ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED;  in find_free_extent()
4302 	trace_find_free_extent(root, ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent()
4303 			       ffe_ctl->flags);  in find_free_extent()
4305 	space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags);  in find_free_extent()
4307 		btrfs_err(fs_info, "No space info for %llu", ffe_ctl->flags);  in find_free_extent()
4311 	ret = prepare_allocation(fs_info, ffe_ctl, space_info, ins);  in find_free_extent()
4315 	ffe_ctl->search_start = max(ffe_ctl->search_start,  in find_free_extent()
4317 	ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte);  in find_free_extent()
4318 	if (ffe_ctl->search_start == ffe_ctl->hint_byte) {  in find_free_extent()
4320 						       ffe_ctl->search_start);  in find_free_extent()
4328 		if (block_group && block_group_bits(block_group, ffe_ctl->flags) &&  in find_free_extent()
4342 				ffe_ctl->index = btrfs_bg_flags_to_raid_index(  in find_free_extent()
4345 						       ffe_ctl->delalloc);  in find_free_extent()
4353 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4354 	if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) ||  in find_free_extent()
4355 	    ffe_ctl->index == 0)  in find_free_extent()
4359 			    &space_info->block_groups[ffe_ctl->index], list) {  in find_free_extent()
4364 			if (ffe_ctl->for_treelog)  in find_free_extent()
4366 			if (ffe_ctl->for_data_reloc)  in find_free_extent()
4371 		btrfs_grab_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4372 		ffe_ctl->search_start = block_group->start;  in find_free_extent()
4379 		if (!block_group_bits(block_group, ffe_ctl->flags)) {  in find_free_extent()
4390 			if ((ffe_ctl->flags & extra) && !(block_group->flags & extra))  in find_free_extent()
4398 			btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4403 		ffe_ctl->cached = btrfs_block_group_done(block_group);  in find_free_extent()
4404 		if (unlikely(!ffe_ctl->cached)) {  in find_free_extent()
4405 			ffe_ctl->have_caching_bg = true;  in find_free_extent()
4428 		ret = do_allocation(block_group, ffe_ctl, &bg_ret);  in find_free_extent()
4432 							  ffe_ctl->delalloc);  in find_free_extent()
4442 		ffe_ctl->search_start = round_up(ffe_ctl->found_offset,  in find_free_extent()
4446 		if (ffe_ctl->search_start + ffe_ctl->num_bytes >  in find_free_extent()
4449 					    ffe_ctl->found_offset,  in find_free_extent()
4450 					    ffe_ctl->num_bytes);  in find_free_extent()
4454 		if (ffe_ctl->found_offset < ffe_ctl->search_start)  in find_free_extent()
4456 					ffe_ctl->found_offset,  in find_free_extent()
4457 					ffe_ctl->search_start - ffe_ctl->found_offset);  in find_free_extent()
4459 		ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes,  in find_free_extent()
4460 					       ffe_ctl->num_bytes,  in find_free_extent()
4461 					       ffe_ctl->delalloc);  in find_free_extent()
4464 					ffe_ctl->found_offset,  in find_free_extent()
4465 					ffe_ctl->num_bytes);  in find_free_extent()
4471 		ins->objectid = ffe_ctl->search_start;  in find_free_extent()
4472 		ins->offset = ffe_ctl->num_bytes;  in find_free_extent()
4474 		trace_btrfs_reserve_extent(block_group, ffe_ctl->search_start,  in find_free_extent()
4475 					   ffe_ctl->num_bytes);  in find_free_extent()
4476 		btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4479 		release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc);  in find_free_extent()
4484 	ret = find_free_extent_update_loop(fs_info, ins, ffe_ctl, full_search);  in find_free_extent()
4493 		if (!ffe_ctl->max_extent_size)  in find_free_extent()
4494 			ffe_ctl->max_extent_size = ffe_ctl->total_free_space;  in find_free_extent()
4496 		space_info->max_extent_size = ffe_ctl->max_extent_size;  in find_free_extent()
4498 		ins->offset = ffe_ctl->max_extent_size;  in find_free_extent()
4556 	struct find_free_extent_ctl ffe_ctl = {};  in btrfs_reserve_extent()  local
4567 	ffe_ctl.ram_bytes = ram_bytes;  in btrfs_reserve_extent()
4568 	ffe_ctl.num_bytes = num_bytes;  in btrfs_reserve_extent()
4569 	ffe_ctl.min_alloc_size = min_alloc_size;  in btrfs_reserve_extent()
4570 	ffe_ctl.empty_size = empty_size;  in btrfs_reserve_extent()
4571 	ffe_ctl.flags = flags;  in btrfs_reserve_extent()
4572 	ffe_ctl.delalloc = delalloc;  in btrfs_reserve_extent()
4573 	ffe_ctl.hint_byte = hint_byte;  in btrfs_reserve_extent()
4574 	ffe_ctl.for_treelog = for_treelog;  in btrfs_reserve_extent()
4575 	ffe_ctl.for_data_reloc = for_data_reloc;  in btrfs_reserve_extent()
4577 	ret = find_free_extent(root, ins, &ffe_ctl);  in btrfs_reserve_extent()