1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Functions related to generic helpers functions
4 */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/scatterlist.h>
10
11 #include "blk.h"
12
bio_discard_limit(struct block_device * bdev,sector_t sector)13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)
14 {
15 unsigned int discard_granularity = bdev_discard_granularity(bdev);
16 sector_t granularity_aligned_sector;
17
18 if (bdev_is_partition(bdev))
19 sector += bdev->bd_start_sect;
20
21 granularity_aligned_sector =
22 round_up(sector, discard_granularity >> SECTOR_SHIFT);
23
24 /*
25 * Make sure subsequent bios start aligned to the discard granularity if
26 * it needs to be split.
27 */
28 if (granularity_aligned_sector != sector)
29 return granularity_aligned_sector - sector;
30
31 /*
32 * Align the bio size to the discard granularity to make splitting the bio
33 * at discard granularity boundaries easier in the driver if needed.
34 */
35 return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT;
36 }
37
__blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop)38 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
39 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)
40 {
41 struct bio *bio = *biop;
42 sector_t bs_mask;
43
44 if (bdev_read_only(bdev))
45 return -EPERM;
46 if (!bdev_max_discard_sectors(bdev))
47 return -EOPNOTSUPP;
48
49 /* In case the discard granularity isn't set by buggy device driver */
50 if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) {
51 char dev_name[BDEVNAME_SIZE];
52
53 bdevname(bdev, dev_name);
54 pr_err_ratelimited("%s: Error: discard_granularity is 0.\n", dev_name);
55 return -EOPNOTSUPP;
56 }
57
58 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
59 if ((sector | nr_sects) & bs_mask)
60 return -EINVAL;
61
62 if (!nr_sects)
63 return -EINVAL;
64
65 while (nr_sects) {
66 sector_t req_sects =
67 min(nr_sects, bio_discard_limit(bdev, sector));
68
69 bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask);
70 bio->bi_iter.bi_sector = sector;
71 bio->bi_iter.bi_size = req_sects << 9;
72 sector += req_sects;
73 nr_sects -= req_sects;
74
75 /*
76 * We can loop for a long time in here, if someone does
77 * full device discards (like mkfs). Be nice and allow
78 * us to schedule out to avoid softlocking if preempt
79 * is disabled.
80 */
81 cond_resched();
82 }
83
84 *biop = bio;
85 return 0;
86 }
87 EXPORT_SYMBOL(__blkdev_issue_discard);
88
89 /**
90 * blkdev_issue_discard - queue a discard
91 * @bdev: blockdev to issue discard for
92 * @sector: start sector
93 * @nr_sects: number of sectors to discard
94 * @gfp_mask: memory allocation flags (for bio_alloc)
95 *
96 * Description:
97 * Issue a discard request for the sectors in question.
98 */
blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask)99 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
100 sector_t nr_sects, gfp_t gfp_mask)
101 {
102 struct bio *bio = NULL;
103 struct blk_plug plug;
104 int ret;
105
106 blk_start_plug(&plug);
107 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);
108 if (!ret && bio) {
109 ret = submit_bio_wait(bio);
110 if (ret == -EOPNOTSUPP)
111 ret = 0;
112 bio_put(bio);
113 }
114 blk_finish_plug(&plug);
115
116 return ret;
117 }
118 EXPORT_SYMBOL(blkdev_issue_discard);
119
__blkdev_issue_write_zeroes(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags)120 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
121 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
122 struct bio **biop, unsigned flags)
123 {
124 struct bio *bio = *biop;
125 unsigned int max_write_zeroes_sectors;
126
127 if (bdev_read_only(bdev))
128 return -EPERM;
129
130 /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
131 max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
132
133 if (max_write_zeroes_sectors == 0)
134 return -EOPNOTSUPP;
135
136 while (nr_sects) {
137 bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
138 bio->bi_iter.bi_sector = sector;
139 if (flags & BLKDEV_ZERO_NOUNMAP)
140 bio->bi_opf |= REQ_NOUNMAP;
141
142 if (nr_sects > max_write_zeroes_sectors) {
143 bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
144 nr_sects -= max_write_zeroes_sectors;
145 sector += max_write_zeroes_sectors;
146 } else {
147 bio->bi_iter.bi_size = nr_sects << 9;
148 nr_sects = 0;
149 }
150 cond_resched();
151 }
152
153 *biop = bio;
154 return 0;
155 }
156
157 /*
158 * Convert a number of 512B sectors to a number of pages.
159 * The result is limited to a number of pages that can fit into a BIO.
160 * Also make sure that the result is always at least 1 (page) for the cases
161 * where nr_sects is lower than the number of sectors in a page.
162 */
__blkdev_sectors_to_bio_pages(sector_t nr_sects)163 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
164 {
165 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
166
167 return min(pages, (sector_t)BIO_MAX_VECS);
168 }
169
__blkdev_issue_zero_pages(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop)170 static int __blkdev_issue_zero_pages(struct block_device *bdev,
171 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
172 struct bio **biop)
173 {
174 struct bio *bio = *biop;
175 int bi_size = 0;
176 unsigned int sz;
177
178 if (bdev_read_only(bdev))
179 return -EPERM;
180
181 while (nr_sects != 0) {
182 bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects),
183 REQ_OP_WRITE, gfp_mask);
184 bio->bi_iter.bi_sector = sector;
185
186 while (nr_sects != 0) {
187 sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
188 bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
189 nr_sects -= bi_size >> 9;
190 sector += bi_size >> 9;
191 if (bi_size < sz)
192 break;
193 }
194 cond_resched();
195 }
196
197 *biop = bio;
198 return 0;
199 }
200
201 /**
202 * __blkdev_issue_zeroout - generate number of zero filed write bios
203 * @bdev: blockdev to issue
204 * @sector: start sector
205 * @nr_sects: number of sectors to write
206 * @gfp_mask: memory allocation flags (for bio_alloc)
207 * @biop: pointer to anchor bio
208 * @flags: controls detailed behavior
209 *
210 * Description:
211 * Zero-fill a block range, either using hardware offload or by explicitly
212 * writing zeroes to the device.
213 *
214 * If a device is using logical block provisioning, the underlying space will
215 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
216 *
217 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
218 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
219 */
__blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags)220 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
221 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
222 unsigned flags)
223 {
224 int ret;
225 sector_t bs_mask;
226
227 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
228 if ((sector | nr_sects) & bs_mask)
229 return -EINVAL;
230
231 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
232 biop, flags);
233 if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
234 return ret;
235
236 return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
237 biop);
238 }
239 EXPORT_SYMBOL(__blkdev_issue_zeroout);
240
241 /**
242 * blkdev_issue_zeroout - zero-fill a block range
243 * @bdev: blockdev to write
244 * @sector: start sector
245 * @nr_sects: number of sectors to write
246 * @gfp_mask: memory allocation flags (for bio_alloc)
247 * @flags: controls detailed behavior
248 *
249 * Description:
250 * Zero-fill a block range, either using hardware offload or by explicitly
251 * writing zeroes to the device. See __blkdev_issue_zeroout() for the
252 * valid values for %flags.
253 */
blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,unsigned flags)254 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
255 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
256 {
257 int ret = 0;
258 sector_t bs_mask;
259 struct bio *bio;
260 struct blk_plug plug;
261 bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
262
263 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
264 if ((sector | nr_sects) & bs_mask)
265 return -EINVAL;
266
267 retry:
268 bio = NULL;
269 blk_start_plug(&plug);
270 if (try_write_zeroes) {
271 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
272 gfp_mask, &bio, flags);
273 } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
274 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
275 gfp_mask, &bio);
276 } else {
277 /* No zeroing offload support */
278 ret = -EOPNOTSUPP;
279 }
280 if (ret == 0 && bio) {
281 ret = submit_bio_wait(bio);
282 bio_put(bio);
283 }
284 blk_finish_plug(&plug);
285 if (ret && try_write_zeroes) {
286 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
287 try_write_zeroes = false;
288 goto retry;
289 }
290 if (!bdev_write_zeroes_sectors(bdev)) {
291 /*
292 * Zeroing offload support was indicated, but the
293 * device reported ILLEGAL REQUEST (for some devices
294 * there is no non-destructive way to verify whether
295 * WRITE ZEROES is actually supported).
296 */
297 ret = -EOPNOTSUPP;
298 }
299 }
300
301 return ret;
302 }
303 EXPORT_SYMBOL(blkdev_issue_zeroout);
304
blkdev_issue_secure_erase(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp)305 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
306 sector_t nr_sects, gfp_t gfp)
307 {
308 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
309 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);
310 struct bio *bio = NULL;
311 struct blk_plug plug;
312 int ret = 0;
313
314 if (max_sectors == 0)
315 return -EOPNOTSUPP;
316 if ((sector | nr_sects) & bs_mask)
317 return -EINVAL;
318 if (bdev_read_only(bdev))
319 return -EPERM;
320
321 blk_start_plug(&plug);
322 for (;;) {
323 unsigned int len = min_t(sector_t, nr_sects, max_sectors);
324
325 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);
326 bio->bi_iter.bi_sector = sector;
327 bio->bi_iter.bi_size = len;
328
329 sector += len << SECTOR_SHIFT;
330 nr_sects -= len << SECTOR_SHIFT;
331 if (!nr_sects) {
332 ret = submit_bio_wait(bio);
333 bio_put(bio);
334 break;
335 }
336 cond_resched();
337 }
338 blk_finish_plug(&plug);
339
340 return ret;
341 }
342 EXPORT_SYMBOL(blkdev_issue_secure_erase);
343