1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Functions related to generic helpers functions
4  */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/scatterlist.h>
10 
11 #include "blk.h"
12 
bio_discard_limit(struct block_device * bdev,sector_t sector)13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)
14 {
15 	unsigned int discard_granularity = bdev_discard_granularity(bdev);
16 	sector_t granularity_aligned_sector;
17 
18 	if (bdev_is_partition(bdev))
19 		sector += bdev->bd_start_sect;
20 
21 	granularity_aligned_sector =
22 		round_up(sector, discard_granularity >> SECTOR_SHIFT);
23 
24 	/*
25 	 * Make sure subsequent bios start aligned to the discard granularity if
26 	 * it needs to be split.
27 	 */
28 	if (granularity_aligned_sector != sector)
29 		return granularity_aligned_sector - sector;
30 
31 	/*
32 	 * Align the bio size to the discard granularity to make splitting the bio
33 	 * at discard granularity boundaries easier in the driver if needed.
34 	 */
35 	return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT;
36 }
37 
__blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop)38 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
39 		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)
40 {
41 	struct bio *bio = *biop;
42 	sector_t bs_mask;
43 
44 	if (bdev_read_only(bdev))
45 		return -EPERM;
46 	if (!bdev_max_discard_sectors(bdev))
47 		return -EOPNOTSUPP;
48 
49 	/* In case the discard granularity isn't set by buggy device driver */
50 	if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) {
51 		char dev_name[BDEVNAME_SIZE];
52 
53 		bdevname(bdev, dev_name);
54 		pr_err_ratelimited("%s: Error: discard_granularity is 0.\n", dev_name);
55 		return -EOPNOTSUPP;
56 	}
57 
58 	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
59 	if ((sector | nr_sects) & bs_mask)
60 		return -EINVAL;
61 
62 	if (!nr_sects)
63 		return -EINVAL;
64 
65 	while (nr_sects) {
66 		sector_t req_sects =
67 			min(nr_sects, bio_discard_limit(bdev, sector));
68 
69 		bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask);
70 		bio->bi_iter.bi_sector = sector;
71 		bio->bi_iter.bi_size = req_sects << 9;
72 		sector += req_sects;
73 		nr_sects -= req_sects;
74 
75 		/*
76 		 * We can loop for a long time in here, if someone does
77 		 * full device discards (like mkfs). Be nice and allow
78 		 * us to schedule out to avoid softlocking if preempt
79 		 * is disabled.
80 		 */
81 		cond_resched();
82 	}
83 
84 	*biop = bio;
85 	return 0;
86 }
87 EXPORT_SYMBOL(__blkdev_issue_discard);
88 
89 /**
90  * blkdev_issue_discard - queue a discard
91  * @bdev:	blockdev to issue discard for
92  * @sector:	start sector
93  * @nr_sects:	number of sectors to discard
94  * @gfp_mask:	memory allocation flags (for bio_alloc)
95  *
96  * Description:
97  *    Issue a discard request for the sectors in question.
98  */
blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask)99 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
100 		sector_t nr_sects, gfp_t gfp_mask)
101 {
102 	struct bio *bio = NULL;
103 	struct blk_plug plug;
104 	int ret;
105 
106 	blk_start_plug(&plug);
107 	ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);
108 	if (!ret && bio) {
109 		ret = submit_bio_wait(bio);
110 		if (ret == -EOPNOTSUPP)
111 			ret = 0;
112 		bio_put(bio);
113 	}
114 	blk_finish_plug(&plug);
115 
116 	return ret;
117 }
118 EXPORT_SYMBOL(blkdev_issue_discard);
119 
__blkdev_issue_write_zeroes(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags)120 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
121 		sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
122 		struct bio **biop, unsigned flags)
123 {
124 	struct bio *bio = *biop;
125 	unsigned int max_write_zeroes_sectors;
126 
127 	if (bdev_read_only(bdev))
128 		return -EPERM;
129 
130 	/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
131 	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
132 
133 	if (max_write_zeroes_sectors == 0)
134 		return -EOPNOTSUPP;
135 
136 	while (nr_sects) {
137 		bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
138 		bio->bi_iter.bi_sector = sector;
139 		if (flags & BLKDEV_ZERO_NOUNMAP)
140 			bio->bi_opf |= REQ_NOUNMAP;
141 
142 		if (nr_sects > max_write_zeroes_sectors) {
143 			bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
144 			nr_sects -= max_write_zeroes_sectors;
145 			sector += max_write_zeroes_sectors;
146 		} else {
147 			bio->bi_iter.bi_size = nr_sects << 9;
148 			nr_sects = 0;
149 		}
150 		cond_resched();
151 	}
152 
153 	*biop = bio;
154 	return 0;
155 }
156 
157 /*
158  * Convert a number of 512B sectors to a number of pages.
159  * The result is limited to a number of pages that can fit into a BIO.
160  * Also make sure that the result is always at least 1 (page) for the cases
161  * where nr_sects is lower than the number of sectors in a page.
162  */
__blkdev_sectors_to_bio_pages(sector_t nr_sects)163 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
164 {
165 	sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
166 
167 	return min(pages, (sector_t)BIO_MAX_VECS);
168 }
169 
__blkdev_issue_zero_pages(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop)170 static int __blkdev_issue_zero_pages(struct block_device *bdev,
171 		sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
172 		struct bio **biop)
173 {
174 	struct bio *bio = *biop;
175 	int bi_size = 0;
176 	unsigned int sz;
177 
178 	if (bdev_read_only(bdev))
179 		return -EPERM;
180 
181 	while (nr_sects != 0) {
182 		bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects),
183 				   REQ_OP_WRITE, gfp_mask);
184 		bio->bi_iter.bi_sector = sector;
185 
186 		while (nr_sects != 0) {
187 			sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
188 			bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
189 			nr_sects -= bi_size >> 9;
190 			sector += bi_size >> 9;
191 			if (bi_size < sz)
192 				break;
193 		}
194 		cond_resched();
195 	}
196 
197 	*biop = bio;
198 	return 0;
199 }
200 
201 /**
202  * __blkdev_issue_zeroout - generate number of zero filed write bios
203  * @bdev:	blockdev to issue
204  * @sector:	start sector
205  * @nr_sects:	number of sectors to write
206  * @gfp_mask:	memory allocation flags (for bio_alloc)
207  * @biop:	pointer to anchor bio
208  * @flags:	controls detailed behavior
209  *
210  * Description:
211  *  Zero-fill a block range, either using hardware offload or by explicitly
212  *  writing zeroes to the device.
213  *
214  *  If a device is using logical block provisioning, the underlying space will
215  *  not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
216  *
217  *  If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
218  *  -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
219  */
__blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags)220 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
221 		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
222 		unsigned flags)
223 {
224 	int ret;
225 	sector_t bs_mask;
226 
227 	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
228 	if ((sector | nr_sects) & bs_mask)
229 		return -EINVAL;
230 
231 	ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
232 			biop, flags);
233 	if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
234 		return ret;
235 
236 	return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
237 					 biop);
238 }
239 EXPORT_SYMBOL(__blkdev_issue_zeroout);
240 
241 /**
242  * blkdev_issue_zeroout - zero-fill a block range
243  * @bdev:	blockdev to write
244  * @sector:	start sector
245  * @nr_sects:	number of sectors to write
246  * @gfp_mask:	memory allocation flags (for bio_alloc)
247  * @flags:	controls detailed behavior
248  *
249  * Description:
250  *  Zero-fill a block range, either using hardware offload or by explicitly
251  *  writing zeroes to the device.  See __blkdev_issue_zeroout() for the
252  *  valid values for %flags.
253  */
blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,unsigned flags)254 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
255 		sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
256 {
257 	int ret = 0;
258 	sector_t bs_mask;
259 	struct bio *bio;
260 	struct blk_plug plug;
261 	bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
262 
263 	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
264 	if ((sector | nr_sects) & bs_mask)
265 		return -EINVAL;
266 
267 retry:
268 	bio = NULL;
269 	blk_start_plug(&plug);
270 	if (try_write_zeroes) {
271 		ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
272 						  gfp_mask, &bio, flags);
273 	} else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
274 		ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
275 						gfp_mask, &bio);
276 	} else {
277 		/* No zeroing offload support */
278 		ret = -EOPNOTSUPP;
279 	}
280 	if (ret == 0 && bio) {
281 		ret = submit_bio_wait(bio);
282 		bio_put(bio);
283 	}
284 	blk_finish_plug(&plug);
285 	if (ret && try_write_zeroes) {
286 		if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
287 			try_write_zeroes = false;
288 			goto retry;
289 		}
290 		if (!bdev_write_zeroes_sectors(bdev)) {
291 			/*
292 			 * Zeroing offload support was indicated, but the
293 			 * device reported ILLEGAL REQUEST (for some devices
294 			 * there is no non-destructive way to verify whether
295 			 * WRITE ZEROES is actually supported).
296 			 */
297 			ret = -EOPNOTSUPP;
298 		}
299 	}
300 
301 	return ret;
302 }
303 EXPORT_SYMBOL(blkdev_issue_zeroout);
304 
blkdev_issue_secure_erase(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp)305 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
306 		sector_t nr_sects, gfp_t gfp)
307 {
308 	sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
309 	unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);
310 	struct bio *bio = NULL;
311 	struct blk_plug plug;
312 	int ret = 0;
313 
314 	if (max_sectors == 0)
315 		return -EOPNOTSUPP;
316 	if ((sector | nr_sects) & bs_mask)
317 		return -EINVAL;
318 	if (bdev_read_only(bdev))
319 		return -EPERM;
320 
321 	blk_start_plug(&plug);
322 	for (;;) {
323 		unsigned int len = min_t(sector_t, nr_sects, max_sectors);
324 
325 		bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);
326 		bio->bi_iter.bi_sector = sector;
327 		bio->bi_iter.bi_size = len;
328 
329 		sector += len << SECTOR_SHIFT;
330 		nr_sects -= len << SECTOR_SHIFT;
331 		if (!nr_sects) {
332 			ret = submit_bio_wait(bio);
333 			bio_put(bio);
334 			break;
335 		}
336 		cond_resched();
337 	}
338 	blk_finish_plug(&plug);
339 
340 	return ret;
341 }
342 EXPORT_SYMBOL(blkdev_issue_secure_erase);
343