1// SPDX-License-Identifier: GPL-2.0
2/*
3 * bio-integrity.c - bio data integrity extensions
4 *
5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
7 */
8
9#include <linux/blk-integrity.h>
10#include "blk.h"
11
12struct bio_integrity_alloc {
13 struct bio_integrity_payload bip;
14 struct bio_vec bvecs[];
15};
16
17static mempool_t integrity_buf_pool;
18
19void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer)
20{
21 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
22 struct bio_integrity_payload *bip = bio_integrity(bio);
23 unsigned int len = bio_integrity_bytes(bi, bio_sectors(bio));
24 gfp_t gfp = GFP_NOIO | (zero_buffer ? __GFP_ZERO : 0);
25 void *buf;
26
27 buf = kmalloc(len, (gfp & ~__GFP_DIRECT_RECLAIM) |
28 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN);
29 if (unlikely(!buf)) {
30 struct page *page;
31
32 page = mempool_alloc(&integrity_buf_pool, GFP_NOFS);
33 if (zero_buffer)
34 memset(page_address(page), 0, len);
35 bvec_set_page(bv: &bip->bip_vec[0], page, len, offset: 0);
36 bip->bip_flags |= BIP_MEMPOOL;
37 } else {
38 bvec_set_page(bv: &bip->bip_vec[0], virt_to_page(buf), len,
39 offset_in_page(buf));
40 }
41
42 bip->bip_vcnt = 1;
43 bip->bip_iter.bi_size = len;
44}
45
46void bio_integrity_free_buf(struct bio_integrity_payload *bip)
47{
48 struct bio_vec *bv = &bip->bip_vec[0];
49
50 if (bip->bip_flags & BIP_MEMPOOL)
51 mempool_free(element: bv->bv_page, pool: &integrity_buf_pool);
52 else
53 kfree(objp: bvec_virt(bvec: bv));
54}
55
56/**
57 * bio_integrity_free - Free bio integrity payload
58 * @bio: bio containing bip to be freed
59 *
60 * Description: Free the integrity portion of a bio.
61 */
62void bio_integrity_free(struct bio *bio)
63{
64 kfree(objp: bio_integrity(bio));
65 bio->bi_integrity = NULL;
66 bio->bi_opf &= ~REQ_INTEGRITY;
67}
68
69void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
70 struct bio_vec *bvecs, unsigned int nr_vecs)
71{
72 memset(bip, 0, sizeof(*bip));
73 bip->bip_max_vcnt = nr_vecs;
74 if (nr_vecs)
75 bip->bip_vec = bvecs;
76
77 bio->bi_integrity = bip;
78 bio->bi_opf |= REQ_INTEGRITY;
79}
80
81/**
82 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
83 * @bio: bio to attach integrity metadata to
84 * @gfp_mask: Memory allocation mask
85 * @nr_vecs: Number of integrity metadata scatter-gather elements
86 *
87 * Description: This function prepares a bio for attaching integrity
88 * metadata. nr_vecs specifies the maximum number of pages containing
89 * integrity metadata that can be attached.
90 */
91struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
92 gfp_t gfp_mask,
93 unsigned int nr_vecs)
94{
95 struct bio_integrity_alloc *bia;
96
97 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
98 return ERR_PTR(error: -EOPNOTSUPP);
99
100 bia = kmalloc(struct_size(bia, bvecs, nr_vecs), gfp_mask);
101 if (unlikely(!bia))
102 return ERR_PTR(error: -ENOMEM);
103 bio_integrity_init(bio, bip: &bia->bip, bvecs: bia->bvecs, nr_vecs);
104 return &bia->bip;
105}
106EXPORT_SYMBOL(bio_integrity_alloc);
107
108static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs)
109{
110 int i;
111
112 for (i = 0; i < nr_vecs; i++)
113 unpin_user_page(page: bv[i].bv_page);
114}
115
116static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
117{
118 unsigned short orig_nr_vecs = bip->bip_max_vcnt - 1;
119 struct bio_vec *orig_bvecs = &bip->bip_vec[1];
120 struct bio_vec *bounce_bvec = &bip->bip_vec[0];
121 size_t bytes = bounce_bvec->bv_len;
122 struct iov_iter orig_iter;
123 int ret;
124
125 iov_iter_bvec(i: &orig_iter, ITER_DEST, bvec: orig_bvecs, nr_segs: orig_nr_vecs, count: bytes);
126 ret = copy_to_iter(addr: bvec_virt(bvec: bounce_bvec), bytes, i: &orig_iter);
127 WARN_ON_ONCE(ret != bytes);
128
129 bio_integrity_unpin_bvec(bv: orig_bvecs, nr_vecs: orig_nr_vecs);
130}
131
132/**
133 * bio_integrity_unmap_user - Unmap user integrity payload
134 * @bio: bio containing bip to be unmapped
135 *
136 * Unmap the user mapped integrity portion of a bio.
137 */
138void bio_integrity_unmap_user(struct bio *bio)
139{
140 struct bio_integrity_payload *bip = bio_integrity(bio);
141
142 if (bip->bip_flags & BIP_COPY_USER) {
143 if (bio_data_dir(bio) == READ)
144 bio_integrity_uncopy_user(bip);
145 kfree(objp: bvec_virt(bvec: bip->bip_vec));
146 return;
147 }
148
149 bio_integrity_unpin_bvec(bv: bip->bip_vec, nr_vecs: bip->bip_max_vcnt);
150}
151
152/**
153 * bio_integrity_add_page - Attach integrity metadata
154 * @bio: bio to update
155 * @page: page containing integrity metadata
156 * @len: number of bytes of integrity metadata in page
157 * @offset: start offset within page
158 *
159 * Description: Attach a page containing integrity metadata to bio.
160 */
161int bio_integrity_add_page(struct bio *bio, struct page *page,
162 unsigned int len, unsigned int offset)
163{
164 struct request_queue *q = bdev_get_queue(bdev: bio->bi_bdev);
165 struct bio_integrity_payload *bip = bio_integrity(bio);
166
167 if (bip->bip_vcnt > 0) {
168 struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1];
169
170 if (!zone_device_pages_have_same_pgmap(a: bv->bv_page, b: page))
171 return 0;
172
173 if (bvec_try_merge_hw_page(q, bv, page, len, offset)) {
174 bip->bip_iter.bi_size += len;
175 return len;
176 }
177
178 if (bip->bip_vcnt >=
179 min(bip->bip_max_vcnt, queue_max_integrity_segments(q)))
180 return 0;
181
182 /*
183 * If the queue doesn't support SG gaps and adding this segment
184 * would create a gap, disallow it.
185 */
186 if (bvec_gap_to_prev(lim: &q->limits, bprv: bv, offset))
187 return 0;
188 }
189
190 bvec_set_page(bv: &bip->bip_vec[bip->bip_vcnt], page, len, offset);
191 bip->bip_vcnt++;
192 bip->bip_iter.bi_size += len;
193
194 return len;
195}
196EXPORT_SYMBOL(bio_integrity_add_page);
197
198static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
199 int nr_vecs, unsigned int len)
200{
201 bool write = op_is_write(op: bio_op(bio));
202 struct bio_integrity_payload *bip;
203 struct iov_iter iter;
204 void *buf;
205 int ret;
206
207 buf = kmalloc(len, GFP_KERNEL);
208 if (!buf)
209 return -ENOMEM;
210
211 if (write) {
212 iov_iter_bvec(i: &iter, ITER_SOURCE, bvec, nr_segs: nr_vecs, count: len);
213 if (!copy_from_iter_full(addr: buf, bytes: len, i: &iter)) {
214 ret = -EFAULT;
215 goto free_buf;
216 }
217
218 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
219 } else {
220 memset(buf, 0, len);
221
222 /*
223 * We need to preserve the original bvec and the number of vecs
224 * in it for completion handling
225 */
226 bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs + 1);
227 }
228
229 if (IS_ERR(ptr: bip)) {
230 ret = PTR_ERR(ptr: bip);
231 goto free_buf;
232 }
233
234 if (write)
235 bio_integrity_unpin_bvec(bv: bvec, nr_vecs);
236 else
237 memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
238
239 ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
240 offset_in_page(buf));
241 if (ret != len) {
242 ret = -ENOMEM;
243 goto free_bip;
244 }
245
246 bip->bip_flags |= BIP_COPY_USER;
247 bip->bip_vcnt = nr_vecs;
248 return 0;
249free_bip:
250 bio_integrity_free(bio);
251free_buf:
252 kfree(objp: buf);
253 return ret;
254}
255
256static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
257 int nr_vecs, unsigned int len)
258{
259 struct bio_integrity_payload *bip;
260
261 bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
262 if (IS_ERR(ptr: bip))
263 return PTR_ERR(ptr: bip);
264
265 memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
266 bip->bip_iter.bi_size = len;
267 bip->bip_vcnt = nr_vecs;
268 return 0;
269}
270
271static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
272 int nr_vecs, ssize_t bytes, ssize_t offset,
273 bool *is_p2p)
274{
275 unsigned int nr_bvecs = 0;
276 int i, j;
277
278 for (i = 0; i < nr_vecs; i = j) {
279 size_t size = min_t(size_t, bytes, PAGE_SIZE - offset);
280 struct folio *folio = page_folio(pages[i]);
281
282 bytes -= size;
283 for (j = i + 1; j < nr_vecs; j++) {
284 size_t next = min_t(size_t, PAGE_SIZE, bytes);
285
286 if (page_folio(pages[j]) != folio ||
287 pages[j] != pages[j - 1] + 1)
288 break;
289 unpin_user_page(page: pages[j]);
290 size += next;
291 bytes -= next;
292 }
293
294 if (is_pci_p2pdma_page(page: pages[i]))
295 *is_p2p = true;
296
297 bvec_set_page(bv: &bvec[nr_bvecs], page: pages[i], len: size, offset);
298 offset = 0;
299 nr_bvecs++;
300 }
301
302 return nr_bvecs;
303}
304
305int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
306{
307 struct request_queue *q = bdev_get_queue(bdev: bio->bi_bdev);
308 struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
309 struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
310 iov_iter_extraction_t extraction_flags = 0;
311 size_t offset, bytes = iter->count;
312 bool copy, is_p2p = false;
313 unsigned int nr_bvecs;
314 int ret, nr_vecs;
315
316 if (bio_integrity(bio))
317 return -EINVAL;
318 if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q))
319 return -E2BIG;
320
321 nr_vecs = iov_iter_npages(i: iter, BIO_MAX_VECS + 1);
322 if (nr_vecs > BIO_MAX_VECS)
323 return -E2BIG;
324 if (nr_vecs > UIO_FASTIOV) {
325 bvec = kcalloc(nr_vecs, sizeof(*bvec), GFP_KERNEL);
326 if (!bvec)
327 return -ENOMEM;
328 pages = NULL;
329 }
330
331 copy = iov_iter_alignment(i: iter) &
332 blk_lim_dma_alignment_and_pad(lim: &q->limits);
333
334 if (blk_queue_pci_p2pdma(q))
335 extraction_flags |= ITER_ALLOW_P2PDMA;
336
337 ret = iov_iter_extract_pages(i: iter, pages: &pages, maxsize: bytes, maxpages: nr_vecs,
338 extraction_flags, offset0: &offset);
339 if (unlikely(ret < 0))
340 goto free_bvec;
341
342 nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset,
343 is_p2p: &is_p2p);
344 if (pages != stack_pages)
345 kvfree(addr: pages);
346 if (nr_bvecs > queue_max_integrity_segments(q))
347 copy = true;
348 if (is_p2p)
349 bio->bi_opf |= REQ_NOMERGE;
350
351 if (copy)
352 ret = bio_integrity_copy_user(bio, bvec, nr_vecs: nr_bvecs, len: bytes);
353 else
354 ret = bio_integrity_init_user(bio, bvec, nr_vecs: nr_bvecs, len: bytes);
355 if (ret)
356 goto release_pages;
357 if (bvec != stack_vec)
358 kfree(objp: bvec);
359
360 return 0;
361
362release_pages:
363 bio_integrity_unpin_bvec(bv: bvec, nr_vecs: nr_bvecs);
364free_bvec:
365 if (bvec != stack_vec)
366 kfree(objp: bvec);
367 return ret;
368}
369
370static void bio_uio_meta_to_bip(struct bio *bio, struct uio_meta *meta)
371{
372 struct bio_integrity_payload *bip = bio_integrity(bio);
373
374 if (meta->flags & IO_INTEGRITY_CHK_GUARD)
375 bip->bip_flags |= BIP_CHECK_GUARD;
376 if (meta->flags & IO_INTEGRITY_CHK_APPTAG)
377 bip->bip_flags |= BIP_CHECK_APPTAG;
378 if (meta->flags & IO_INTEGRITY_CHK_REFTAG)
379 bip->bip_flags |= BIP_CHECK_REFTAG;
380
381 bip->app_tag = meta->app_tag;
382}
383
384int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
385{
386 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
387 unsigned int integrity_bytes;
388 int ret;
389 struct iov_iter it;
390
391 if (!bi)
392 return -EINVAL;
393 /*
394 * original meta iterator can be bigger.
395 * process integrity info corresponding to current data buffer only.
396 */
397 it = meta->iter;
398 integrity_bytes = bio_integrity_bytes(bi, bio_sectors(bio));
399 if (it.count < integrity_bytes)
400 return -EINVAL;
401
402 /* should fit into two bytes */
403 BUILD_BUG_ON(IO_INTEGRITY_VALID_FLAGS >= (1 << 16));
404
405 if (meta->flags && (meta->flags & ~IO_INTEGRITY_VALID_FLAGS))
406 return -EINVAL;
407
408 it.count = integrity_bytes;
409 ret = bio_integrity_map_user(bio, iter: &it);
410 if (!ret) {
411 bio_uio_meta_to_bip(bio, meta);
412 bip_set_seed(bip: bio_integrity(bio), seed: meta->seed);
413 iov_iter_advance(i: &meta->iter, bytes: integrity_bytes);
414 meta->seed += bio_integrity_intervals(bi, bio_sectors(bio));
415 }
416 return ret;
417}
418
419/**
420 * bio_integrity_advance - Advance integrity vector
421 * @bio: bio whose integrity vector to update
422 * @bytes_done: number of data bytes that have been completed
423 *
424 * Description: This function calculates how many integrity bytes the
425 * number of completed data bytes correspond to and advances the
426 * integrity vector accordingly.
427 */
428void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
429{
430 struct bio_integrity_payload *bip = bio_integrity(bio);
431 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
432 unsigned bytes = bio_integrity_bytes(bi, sectors: bytes_done >> 9);
433
434 bip->bip_iter.bi_sector += bio_integrity_intervals(bi, sectors: bytes_done >> 9);
435 bvec_iter_advance(bv: bip->bip_vec, iter: &bip->bip_iter, bytes);
436}
437
438/**
439 * bio_integrity_trim - Trim integrity vector
440 * @bio: bio whose integrity vector to update
441 *
442 * Description: Used to trim the integrity vector in a cloned bio.
443 */
444void bio_integrity_trim(struct bio *bio)
445{
446 struct bio_integrity_payload *bip = bio_integrity(bio);
447 struct blk_integrity *bi = blk_get_integrity(disk: bio->bi_bdev->bd_disk);
448
449 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
450}
451EXPORT_SYMBOL(bio_integrity_trim);
452
453/**
454 * bio_integrity_clone - Callback for cloning bios with integrity metadata
455 * @bio: New bio
456 * @bio_src: Original bio
457 * @gfp_mask: Memory allocation mask
458 *
459 * Description: Called to allocate a bip when cloning a bio
460 */
461int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
462 gfp_t gfp_mask)
463{
464 struct bio_integrity_payload *bip_src = bio_integrity(bio: bio_src);
465 struct bio_integrity_payload *bip;
466
467 BUG_ON(bip_src == NULL);
468
469 bip = bio_integrity_alloc(bio, gfp_mask, 0);
470 if (IS_ERR(ptr: bip))
471 return PTR_ERR(ptr: bip);
472
473 bip->bip_vec = bip_src->bip_vec;
474 bip->bip_iter = bip_src->bip_iter;
475 bip->bip_flags = bip_src->bip_flags & BIP_CLONE_FLAGS;
476 bip->app_tag = bip_src->app_tag;
477
478 return 0;
479}
480
481static int __init bio_integrity_initfn(void)
482{
483 if (mempool_init_page_pool(&integrity_buf_pool, BIO_POOL_SIZE,
484 get_order(BLK_INTEGRITY_MAX_SIZE)))
485 panic(fmt: "bio: can't create integrity buf pool\n");
486 return 0;
487}
488subsys_initcall(bio_integrity_initfn);
489

source code of linux/block/bio-integrity.c