3533 				      struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_clustered()  argument
3537 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_clustered()
3542 	cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);  in find_free_extent_clustered()
3546 	    !block_group_bits(cluster_bg, ffe_ctl->flags)))  in find_free_extent_clustered()
3550 			ffe_ctl->num_bytes, cluster_bg->start,  in find_free_extent_clustered()
3551 			&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3556 				ffe_ctl->search_start, ffe_ctl->num_bytes);  in find_free_extent_clustered()
3558 		ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3575 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {  in find_free_extent_clustered()
3577 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3585 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3588 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {  in find_free_extent_clustered()
3594 			ffe_ctl->empty_cluster + ffe_ctl->empty_size,  in find_free_extent_clustered()
3596 	ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,  in find_free_extent_clustered()
3597 			ffe_ctl->num_bytes, aligned_cluster);  in find_free_extent_clustered()
3601 				ffe_ctl->num_bytes, ffe_ctl->search_start,  in find_free_extent_clustered()
3602 				&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3607 					ffe_ctl->search_start,  in find_free_extent_clustered()
3608 					ffe_ctl->num_bytes);  in find_free_extent_clustered()
3609 			ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3612 	} else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&  in find_free_extent_clustered()
3613 		   !ffe_ctl->retry_clustered) {  in find_free_extent_clustered()
3616 		ffe_ctl->retry_clustered = true;  in find_free_extent_clustered()
3617 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_clustered()
3618 				ffe_ctl->empty_cluster + ffe_ctl->empty_size);  in find_free_extent_clustered()
3637 					struct find_free_extent_ctl *ffe_ctl)  in find_free_extent_unclustered()  argument
3639 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_unclustered()
3652 	if (ffe_ctl->cached) {  in find_free_extent_unclustered()
3658 		    ffe_ctl->num_bytes + ffe_ctl->empty_cluster +  in find_free_extent_unclustered()
3659 		    ffe_ctl->empty_size) {  in find_free_extent_unclustered()
3660 			ffe_ctl->total_free_space = max_t(u64,  in find_free_extent_unclustered()
3661 					ffe_ctl->total_free_space,  in find_free_extent_unclustered()
3669 	offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,  in find_free_extent_unclustered()
3670 			ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent_unclustered()
3671 			&ffe_ctl->max_extent_size);  in find_free_extent_unclustered()
3682 	if (!offset && !ffe_ctl->retry_unclustered && !ffe_ctl->cached &&  in find_free_extent_unclustered()
3683 	    ffe_ctl->loop > LOOP_CACHING_NOWAIT) {  in find_free_extent_unclustered()
3684 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_unclustered()
3685 						      ffe_ctl->empty_size);  in find_free_extent_unclustered()
3686 		ffe_ctl->retry_unclustered = true;  in find_free_extent_unclustered()
3691 	ffe_ctl->found_offset = offset;  in find_free_extent_unclustered()
3696 				   struct find_free_extent_ctl *ffe_ctl,  in do_allocation_clustered()  argument
3702 	if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) {  in do_allocation_clustered()
3703 		ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation_clustered()
3709 	return find_free_extent_unclustered(block_group, ffe_ctl);  in do_allocation_clustered()
3734 			       struct find_free_extent_ctl *ffe_ctl,  in do_allocation_zoned()  argument
3741 	u64 num_bytes = ffe_ctl->num_bytes;  in do_allocation_zoned()
3757 	if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) ||  in do_allocation_zoned()
3758 			   (!ffe_ctl->for_treelog && bytenr == log_bytenr)))  in do_allocation_zoned()
3771 	    ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) ||  in do_allocation_zoned()
3772 	     (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr)))  in do_allocation_zoned()
3805 	ASSERT(!ffe_ctl->for_treelog ||  in do_allocation_zoned()
3808 	ASSERT(!ffe_ctl->for_data_reloc ||  in do_allocation_zoned()
3822 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg &&  in do_allocation_zoned()
3832 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg &&  in do_allocation_zoned()
3841 		if (ffe_ctl->max_extent_size < avail) {  in do_allocation_zoned()
3846 			ffe_ctl->max_extent_size = avail;  in do_allocation_zoned()
3847 			ffe_ctl->total_free_space = avail;  in do_allocation_zoned()
3853 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg)  in do_allocation_zoned()
3856 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg)  in do_allocation_zoned()
3859 	ffe_ctl->found_offset = start + block_group->alloc_offset;  in do_allocation_zoned()
3870 	ffe_ctl->search_start = ffe_ctl->found_offset;  in do_allocation_zoned()
3873 	if (ret && ffe_ctl->for_treelog)  in do_allocation_zoned()
3875 	if (ret && ffe_ctl->for_data_reloc &&  in do_allocation_zoned()
3901 			 struct find_free_extent_ctl *ffe_ctl,  in do_allocation()  argument
3904 	switch (ffe_ctl->policy) {  in do_allocation()
3906 		return do_allocation_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation()
3908 		return do_allocation_zoned(block_group, ffe_ctl, bg_ret);  in do_allocation()
3915 				struct find_free_extent_ctl *ffe_ctl,  in release_block_group()  argument
3918 	switch (ffe_ctl->policy) {  in release_block_group()
3920 		ffe_ctl->retry_clustered = false;  in release_block_group()
3921 		ffe_ctl->retry_unclustered = false;  in release_block_group()
3931 	       ffe_ctl->index);  in release_block_group()
3935 static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl,  in found_extent_clustered()  argument
3938 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in found_extent_clustered()
3940 	if (!ffe_ctl->use_cluster && last_ptr) {  in found_extent_clustered()
3947 static void found_extent(struct find_free_extent_ctl *ffe_ctl,  in found_extent()  argument
3950 	switch (ffe_ctl->policy) {  in found_extent()
3952 		found_extent_clustered(ffe_ctl, ins);  in found_extent()
3963 				    struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk_zoned()  argument
3966 	if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))  in can_allocate_chunk_zoned()
3976 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {  in can_allocate_chunk_zoned()
3990 	if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size)  in can_allocate_chunk_zoned()
4000 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)  in can_allocate_chunk_zoned()
4012 			      struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk()  argument
4014 	switch (ffe_ctl->policy) {  in can_allocate_chunk()
4018 		return can_allocate_chunk_zoned(fs_info, ffe_ctl);  in can_allocate_chunk()
4024 static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl)  in chunk_allocation_failed()  argument
4026 	switch (ffe_ctl->policy) {  in chunk_allocation_failed()
4032 		ffe_ctl->loop = LOOP_NO_EMPTY_SIZE;  in chunk_allocation_failed()
4049 					struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_update_loop()  argument
4055 	if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&  in find_free_extent_update_loop()
4056 	    ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)  in find_free_extent_update_loop()
4057 		ffe_ctl->orig_have_caching_bg = true;  in find_free_extent_update_loop()
4060 		found_extent(ffe_ctl, ins);  in find_free_extent_update_loop()
4064 	if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg)  in find_free_extent_update_loop()
4067 	ffe_ctl->index++;  in find_free_extent_update_loop()
4068 	if (ffe_ctl->index < BTRFS_NR_RAID_TYPES)  in find_free_extent_update_loop()
4079 	if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4080 		ffe_ctl->index = 0;  in find_free_extent_update_loop()
4081 		if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) {  in find_free_extent_update_loop()
4087 			if (ffe_ctl->orig_have_caching_bg || !full_search)  in find_free_extent_update_loop()
4088 				ffe_ctl->loop = LOOP_CACHING_WAIT;  in find_free_extent_update_loop()
4090 				ffe_ctl->loop = LOOP_ALLOC_CHUNK;  in find_free_extent_update_loop()
4092 			ffe_ctl->loop++;  in find_free_extent_update_loop()
4095 		if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {  in find_free_extent_update_loop()
4100 			ret = can_allocate_chunk(fs_info, ffe_ctl);  in find_free_extent_update_loop()
4115 			ret = btrfs_chunk_alloc(trans, ffe_ctl->flags,  in find_free_extent_update_loop()
4120 				ret = chunk_allocation_failed(ffe_ctl);  in find_free_extent_update_loop()
4131 		if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4132 			if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED)  in find_free_extent_update_loop()
4139 			if (ffe_ctl->empty_size == 0 &&  in find_free_extent_update_loop()
4140 			    ffe_ctl->empty_cluster == 0)  in find_free_extent_update_loop()
4142 			ffe_ctl->empty_size = 0;  in find_free_extent_update_loop()
4143 			ffe_ctl->empty_cluster = 0;  in find_free_extent_update_loop()
4151 					struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation_clustered()  argument
4168 		    ffe_ctl->num_bytes > space_info->max_extent_size) {  in prepare_allocation_clustered()
4173 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4178 	ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info,  in prepare_allocation_clustered()
4179 					       &ffe_ctl->empty_cluster);  in prepare_allocation_clustered()
4180 	if (ffe_ctl->last_ptr) {  in prepare_allocation_clustered()
4181 		struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in prepare_allocation_clustered()
4185 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4192 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4193 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4202 			      struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation()  argument
4206 	switch (ffe_ctl->policy) {  in prepare_allocation()
4208 		return prepare_allocation_clustered(fs_info, ffe_ctl,  in prepare_allocation()
4211 		if (ffe_ctl->for_treelog) {  in prepare_allocation()
4214 				ffe_ctl->hint_byte = fs_info->treelog_bg;  in prepare_allocation()
4217 		if (ffe_ctl->for_data_reloc) {  in prepare_allocation()
4220 				ffe_ctl->hint_byte = fs_info->data_reloc_bg;  in prepare_allocation()
4256 				     struct find_free_extent_ctl *ffe_ctl)  in find_free_extent()  argument
4265 	WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize);  in find_free_extent()
4267 	ffe_ctl->search_start = 0;  in find_free_extent()
4269 	ffe_ctl->empty_cluster = 0;  in find_free_extent()
4270 	ffe_ctl->last_ptr = NULL;  in find_free_extent()
4271 	ffe_ctl->use_cluster = true;  in find_free_extent()
4272 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4273 	ffe_ctl->orig_have_caching_bg = false;  in find_free_extent()
4274 	ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags);  in find_free_extent()
4275 	ffe_ctl->loop = 0;  in find_free_extent()
4277 	ffe_ctl->retry_clustered = false;  in find_free_extent()
4278 	ffe_ctl->retry_unclustered = false;  in find_free_extent()
4279 	ffe_ctl->cached = 0;  in find_free_extent()
4280 	ffe_ctl->max_extent_size = 0;  in find_free_extent()
4281 	ffe_ctl->total_free_space = 0;  in find_free_extent()
4282 	ffe_ctl->found_offset = 0;  in find_free_extent()
4283 	ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED;  in find_free_extent()
4286 		ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED;  in find_free_extent()
4292 	trace_find_free_extent(root, ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent()
4293 			       ffe_ctl->flags);  in find_free_extent()
4295 	space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags);  in find_free_extent()
4297 		btrfs_err(fs_info, "No space info for %llu", ffe_ctl->flags);  in find_free_extent()
4301 	ret = prepare_allocation(fs_info, ffe_ctl, space_info, ins);  in find_free_extent()
4305 	ffe_ctl->search_start = max(ffe_ctl->search_start,  in find_free_extent()
4307 	ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte);  in find_free_extent()
4308 	if (ffe_ctl->search_start == ffe_ctl->hint_byte) {  in find_free_extent()
4310 						       ffe_ctl->search_start);  in find_free_extent()
4318 		if (block_group && block_group_bits(block_group, ffe_ctl->flags) &&  in find_free_extent()
4332 				ffe_ctl->index = btrfs_bg_flags_to_raid_index(  in find_free_extent()
4335 						       ffe_ctl->delalloc);  in find_free_extent()
4343 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4344 	if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) ||  in find_free_extent()
4345 	    ffe_ctl->index == 0)  in find_free_extent()
4349 			    &space_info->block_groups[ffe_ctl->index], list) {  in find_free_extent()
4354 			if (ffe_ctl->for_treelog)  in find_free_extent()
4356 			if (ffe_ctl->for_data_reloc)  in find_free_extent()
4361 		btrfs_grab_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4362 		ffe_ctl->search_start = block_group->start;  in find_free_extent()
4369 		if (!block_group_bits(block_group, ffe_ctl->flags)) {  in find_free_extent()
4380 			if ((ffe_ctl->flags & extra) && !(block_group->flags & extra))  in find_free_extent()
4388 			btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4393 		ffe_ctl->cached = btrfs_block_group_done(block_group);  in find_free_extent()
4394 		if (unlikely(!ffe_ctl->cached)) {  in find_free_extent()
4395 			ffe_ctl->have_caching_bg = true;  in find_free_extent()
4418 		ret = do_allocation(block_group, ffe_ctl, &bg_ret);  in find_free_extent()
4422 							  ffe_ctl->delalloc);  in find_free_extent()
4432 		ffe_ctl->search_start = round_up(ffe_ctl->found_offset,  in find_free_extent()
4436 		if (ffe_ctl->search_start + ffe_ctl->num_bytes >  in find_free_extent()
4439 					    ffe_ctl->found_offset,  in find_free_extent()
4440 					    ffe_ctl->num_bytes);  in find_free_extent()
4444 		if (ffe_ctl->found_offset < ffe_ctl->search_start)  in find_free_extent()
4446 					ffe_ctl->found_offset,  in find_free_extent()
4447 					ffe_ctl->search_start - ffe_ctl->found_offset);  in find_free_extent()
4449 		ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes,  in find_free_extent()
4450 					       ffe_ctl->num_bytes,  in find_free_extent()
4451 					       ffe_ctl->delalloc);  in find_free_extent()
4454 					ffe_ctl->found_offset,  in find_free_extent()
4455 					ffe_ctl->num_bytes);  in find_free_extent()
4461 		ins->objectid = ffe_ctl->search_start;  in find_free_extent()
4462 		ins->offset = ffe_ctl->num_bytes;  in find_free_extent()
4464 		trace_btrfs_reserve_extent(block_group, ffe_ctl->search_start,  in find_free_extent()
4465 					   ffe_ctl->num_bytes);  in find_free_extent()
4466 		btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4469 		release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc);  in find_free_extent()
4474 	ret = find_free_extent_update_loop(fs_info, ins, ffe_ctl, full_search);  in find_free_extent()
4483 		if (!ffe_ctl->max_extent_size)  in find_free_extent()
4484 			ffe_ctl->max_extent_size = ffe_ctl->total_free_space;  in find_free_extent()
4486 		space_info->max_extent_size = ffe_ctl->max_extent_size;  in find_free_extent()
4488 		ins->offset = ffe_ctl->max_extent_size;  in find_free_extent()
4546 	struct find_free_extent_ctl ffe_ctl = {};  in btrfs_reserve_extent()  local
4557 	ffe_ctl.ram_bytes = ram_bytes;  in btrfs_reserve_extent()
4558 	ffe_ctl.num_bytes = num_bytes;  in btrfs_reserve_extent()
4559 	ffe_ctl.min_alloc_size = min_alloc_size;  in btrfs_reserve_extent()
4560 	ffe_ctl.empty_size = empty_size;  in btrfs_reserve_extent()
4561 	ffe_ctl.flags = flags;  in btrfs_reserve_extent()
4562 	ffe_ctl.delalloc = delalloc;  in btrfs_reserve_extent()
4563 	ffe_ctl.hint_byte = hint_byte;  in btrfs_reserve_extent()
4564 	ffe_ctl.for_treelog = for_treelog;  in btrfs_reserve_extent()
4565 	ffe_ctl.for_data_reloc = for_data_reloc;  in btrfs_reserve_extent()
4567 	ret = find_free_extent(root, ins, &ffe_ctl);  in btrfs_reserve_extent()