1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <net/checksum.h>
8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
22 while (unlikely(!left && n)) { \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
27 __v.iov_base = __p->iov_base; \
29 __v.iov_len -= left; \
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
43 skip += __v.iov_len; \
46 while (unlikely(n)) { \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
51 __v.iov_base = __p->iov_base; \
59 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
60 struct bvec_iter __start; \
61 __start.bi_size = n; \
62 __start.bi_bvec_done = skip; \
64 for_each_bvec(__v, i->bvec, __bi, __start) { \
71 #define iterate_all_kinds(i, n, v, I, B, K) { \
72 size_t skip = i->iov_offset; \
73 if (unlikely(i->type & ITER_BVEC)) { \
75 struct bvec_iter __bi; \
76 iterate_bvec(i, n, v, __bi, skip, (B)) \
77 } else if (unlikely(i->type & ITER_KVEC)) { \
78 const struct kvec *kvec; \
80 iterate_kvec(i, n, v, kvec, skip, (K)) \
82 const struct iovec *iov; \
84 iterate_iovec(i, n, v, iov, skip, (I)) \
88 #define iterate_and_advance(i, n, v, I, B, K) { \
89 if (unlikely(i->count < n)) \
92 size_t skip = i->iov_offset; \
93 if (unlikely(i->type & ITER_BVEC)) { \
94 const struct bio_vec *bvec = i->bvec; \
96 struct bvec_iter __bi; \
97 iterate_bvec(i, n, v, __bi, skip, (B)) \
98 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
99 i->nr_segs -= i->bvec - bvec; \
100 skip = __bi.bi_bvec_done; \
101 } else if (unlikely(i->type & ITER_KVEC)) { \
102 const struct kvec *kvec; \
104 iterate_kvec(i, n, v, kvec, skip, (K)) \
105 if (skip == kvec->iov_len) { \
109 i->nr_segs -= kvec - i->kvec; \
112 const struct iovec *iov; \
114 iterate_iovec(i, n, v, iov, skip, (I)) \
115 if (skip == iov->iov_len) { \
119 i->nr_segs -= iov - i->iov; \
123 i->iov_offset = skip; \
127 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
130 size_t skip, copy, left, wanted;
131 const struct iovec *iov;
135 if (unlikely(bytes > i->count))
138 if (unlikely(!bytes))
143 skip = i->iov_offset;
144 buf = iov->iov_base + skip;
145 copy = min(bytes, iov->iov_len - skip);
147 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
148 kaddr = kmap_atomic(page);
149 from = kaddr + offset;
151 /* first chunk, usually the only one */
152 left = __copy_to_user_inatomic(buf, from, copy);
158 while (unlikely(!left && bytes)) {
161 copy = min(bytes, iov->iov_len);
162 left = __copy_to_user_inatomic(buf, from, copy);
168 if (likely(!bytes)) {
169 kunmap_atomic(kaddr);
172 offset = from - kaddr;
174 kunmap_atomic(kaddr);
175 copy = min(bytes, iov->iov_len - skip);
177 /* Too bad - revert to non-atomic kmap */
180 from = kaddr + offset;
181 left = __copy_to_user(buf, from, copy);
186 while (unlikely(!left && bytes)) {
189 copy = min(bytes, iov->iov_len);
190 left = __copy_to_user(buf, from, copy);
199 if (skip == iov->iov_len) {
203 i->count -= wanted - bytes;
204 i->nr_segs -= iov - i->iov;
206 i->iov_offset = skip;
207 return wanted - bytes;
210 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
213 size_t skip, copy, left, wanted;
214 const struct iovec *iov;
218 if (unlikely(bytes > i->count))
221 if (unlikely(!bytes))
226 skip = i->iov_offset;
227 buf = iov->iov_base + skip;
228 copy = min(bytes, iov->iov_len - skip);
230 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
231 kaddr = kmap_atomic(page);
234 /* first chunk, usually the only one */
235 left = __copy_from_user_inatomic(to, buf, copy);
241 while (unlikely(!left && bytes)) {
244 copy = min(bytes, iov->iov_len);
245 left = __copy_from_user_inatomic(to, buf, copy);
251 if (likely(!bytes)) {
252 kunmap_atomic(kaddr);
257 kunmap_atomic(kaddr);
258 copy = min(bytes, iov->iov_len - skip);
260 /* Too bad - revert to non-atomic kmap */
264 left = __copy_from_user(to, buf, copy);
269 while (unlikely(!left && bytes)) {
272 copy = min(bytes, iov->iov_len);
273 left = __copy_from_user(to, buf, copy);
282 if (skip == iov->iov_len) {
286 i->count -= wanted - bytes;
287 i->nr_segs -= iov - i->iov;
289 i->iov_offset = skip;
290 return wanted - bytes;
294 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
295 * bytes. For each iovec, fault in each page that constitutes the iovec.
297 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
298 * because it is an invalid address).
300 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
302 size_t skip = i->iov_offset;
303 const struct iovec *iov;
307 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
308 iterate_iovec(i, bytes, v, iov, skip, ({
309 err = fault_in_pages_readable(v.iov_base, v.iov_len);
316 EXPORT_SYMBOL(iov_iter_fault_in_readable);
318 void iov_iter_init(struct iov_iter *i, int direction,
319 const struct iovec *iov, unsigned long nr_segs,
322 /* It will get better. Eventually... */
323 if (segment_eq(get_fs(), KERNEL_DS)) {
324 direction |= ITER_KVEC;
326 i->kvec = (struct kvec *)iov;
331 i->nr_segs = nr_segs;
335 EXPORT_SYMBOL(iov_iter_init);
337 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
339 char *from = kmap_atomic(page);
340 memcpy(to, from + offset, len);
344 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
346 char *to = kmap_atomic(page);
347 memcpy(to + offset, from, len);
351 static void memzero_page(struct page *page, size_t offset, size_t len)
353 char *addr = kmap_atomic(page);
354 memset(addr + offset, 0, len);
358 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
360 const char *from = addr;
361 iterate_and_advance(i, bytes, v,
362 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
364 memcpy_to_page(v.bv_page, v.bv_offset,
365 (from += v.bv_len) - v.bv_len, v.bv_len),
366 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
371 EXPORT_SYMBOL(copy_to_iter);
373 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
376 iterate_and_advance(i, bytes, v,
377 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
379 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
380 v.bv_offset, v.bv_len),
381 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
386 EXPORT_SYMBOL(copy_from_iter);
388 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
391 iterate_and_advance(i, bytes, v,
392 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
393 v.iov_base, v.iov_len),
394 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
395 v.bv_offset, v.bv_len),
396 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
401 EXPORT_SYMBOL(copy_from_iter_nocache);
403 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
406 if (i->type & (ITER_BVEC|ITER_KVEC)) {
407 void *kaddr = kmap_atomic(page);
408 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
409 kunmap_atomic(kaddr);
412 return copy_page_to_iter_iovec(page, offset, bytes, i);
414 EXPORT_SYMBOL(copy_page_to_iter);
416 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
419 if (i->type & (ITER_BVEC|ITER_KVEC)) {
420 void *kaddr = kmap_atomic(page);
421 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
422 kunmap_atomic(kaddr);
425 return copy_page_from_iter_iovec(page, offset, bytes, i);
427 EXPORT_SYMBOL(copy_page_from_iter);
429 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
431 iterate_and_advance(i, bytes, v,
432 __clear_user(v.iov_base, v.iov_len),
433 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
434 memset(v.iov_base, 0, v.iov_len)
439 EXPORT_SYMBOL(iov_iter_zero);
441 size_t iov_iter_copy_from_user_atomic(struct page *page,
442 struct iov_iter *i, unsigned long offset, size_t bytes)
444 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
445 iterate_all_kinds(i, bytes, v,
446 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
447 v.iov_base, v.iov_len),
448 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
449 v.bv_offset, v.bv_len),
450 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
452 kunmap_atomic(kaddr);
455 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
457 void iov_iter_advance(struct iov_iter *i, size_t size)
459 iterate_and_advance(i, size, v, 0, 0, 0)
461 EXPORT_SYMBOL(iov_iter_advance);
464 * Return the count of just the current iov_iter segment.
466 size_t iov_iter_single_seg_count(const struct iov_iter *i)
470 else if (i->type & ITER_BVEC)
471 return min(i->count, i->bvec->bv_len - i->iov_offset);
473 return min(i->count, i->iov->iov_len - i->iov_offset);
475 EXPORT_SYMBOL(iov_iter_single_seg_count);
477 void iov_iter_kvec(struct iov_iter *i, int direction,
478 const struct kvec *kvec, unsigned long nr_segs,
481 BUG_ON(!(direction & ITER_KVEC));
484 i->nr_segs = nr_segs;
488 EXPORT_SYMBOL(iov_iter_kvec);
490 void iov_iter_bvec(struct iov_iter *i, int direction,
491 const struct bio_vec *bvec, unsigned long nr_segs,
494 BUG_ON(!(direction & ITER_BVEC));
497 i->nr_segs = nr_segs;
501 EXPORT_SYMBOL(iov_iter_bvec);
503 unsigned long iov_iter_alignment(const struct iov_iter *i)
505 unsigned long res = 0;
506 size_t size = i->count;
511 iterate_all_kinds(i, size, v,
512 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
513 res |= v.bv_offset | v.bv_len,
514 res |= (unsigned long)v.iov_base | v.iov_len
518 EXPORT_SYMBOL(iov_iter_alignment);
520 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
522 unsigned long res = 0;
523 size_t size = i->count;
527 iterate_all_kinds(i, size, v,
528 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
529 (size != v.iov_len ? size : 0), 0),
530 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
531 (size != v.bv_len ? size : 0)),
532 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
533 (size != v.iov_len ? size : 0))
537 EXPORT_SYMBOL(iov_iter_gap_alignment);
539 ssize_t iov_iter_get_pages(struct iov_iter *i,
540 struct page **pages, size_t maxsize, unsigned maxpages,
543 if (maxsize > i->count)
549 iterate_all_kinds(i, maxsize, v, ({
550 unsigned long addr = (unsigned long)v.iov_base;
551 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
555 if (len > maxpages * PAGE_SIZE)
556 len = maxpages * PAGE_SIZE;
557 addr &= ~(PAGE_SIZE - 1);
558 n = DIV_ROUND_UP(len, PAGE_SIZE);
559 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
560 if (unlikely(res < 0))
562 return (res == n ? len : res * PAGE_SIZE) - *start;
564 /* can't be more than PAGE_SIZE */
565 *start = v.bv_offset;
566 get_page(*pages = v.bv_page);
574 EXPORT_SYMBOL(iov_iter_get_pages);
576 static struct page **get_pages_array(size_t n)
578 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
580 p = vmalloc(n * sizeof(struct page *));
584 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
585 struct page ***pages, size_t maxsize,
590 if (maxsize > i->count)
596 iterate_all_kinds(i, maxsize, v, ({
597 unsigned long addr = (unsigned long)v.iov_base;
598 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
602 addr &= ~(PAGE_SIZE - 1);
603 n = DIV_ROUND_UP(len, PAGE_SIZE);
604 p = get_pages_array(n);
607 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
608 if (unlikely(res < 0)) {
613 return (res == n ? len : res * PAGE_SIZE) - *start;
615 /* can't be more than PAGE_SIZE */
616 *start = v.bv_offset;
617 *pages = p = get_pages_array(1);
620 get_page(*p = v.bv_page);
628 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
630 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
637 iterate_and_advance(i, bytes, v, ({
639 next = csum_and_copy_from_user(v.iov_base,
640 (to += v.iov_len) - v.iov_len,
643 sum = csum_block_add(sum, next, off);
648 char *p = kmap_atomic(v.bv_page);
649 next = csum_partial_copy_nocheck(p + v.bv_offset,
650 (to += v.bv_len) - v.bv_len,
653 sum = csum_block_add(sum, next, off);
656 next = csum_partial_copy_nocheck(v.iov_base,
657 (to += v.iov_len) - v.iov_len,
659 sum = csum_block_add(sum, next, off);
666 EXPORT_SYMBOL(csum_and_copy_from_iter);
668 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
671 const char *from = addr;
675 iterate_and_advance(i, bytes, v, ({
677 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
681 sum = csum_block_add(sum, next, off);
686 char *p = kmap_atomic(v.bv_page);
687 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
691 sum = csum_block_add(sum, next, off);
694 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
697 sum = csum_block_add(sum, next, off);
704 EXPORT_SYMBOL(csum_and_copy_to_iter);
706 int iov_iter_npages(const struct iov_iter *i, int maxpages)
708 size_t size = i->count;
714 iterate_all_kinds(i, size, v, ({
715 unsigned long p = (unsigned long)v.iov_base;
716 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
718 if (npages >= maxpages)
722 if (npages >= maxpages)
725 unsigned long p = (unsigned long)v.iov_base;
726 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
728 if (npages >= maxpages)
734 EXPORT_SYMBOL(iov_iter_npages);
736 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
739 if (new->type & ITER_BVEC)
740 return new->bvec = kmemdup(new->bvec,
741 new->nr_segs * sizeof(struct bio_vec),
744 /* iovec and kvec have identical layout */
745 return new->iov = kmemdup(new->iov,
746 new->nr_segs * sizeof(struct iovec),
749 EXPORT_SYMBOL(dup_iter);
751 int import_iovec(int type, const struct iovec __user * uvector,
752 unsigned nr_segs, unsigned fast_segs,
753 struct iovec **iov, struct iov_iter *i)
757 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
765 iov_iter_init(i, type, p, nr_segs, n);
766 *iov = p == *iov ? NULL : p;
769 EXPORT_SYMBOL(import_iovec);
772 #include <linux/compat.h>
774 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
775 unsigned nr_segs, unsigned fast_segs,
776 struct iovec **iov, struct iov_iter *i)
780 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
788 iov_iter_init(i, type, p, nr_segs, n);
789 *iov = p == *iov ? NULL : p;
794 int import_single_range(int rw, void __user *buf, size_t len,
795 struct iovec *iov, struct iov_iter *i)
797 if (len > MAX_RW_COUNT)
799 if (unlikely(!access_ok(!rw, buf, len)))
804 iov_iter_init(i, rw, iov, 1, len);
807 EXPORT_SYMBOL(import_single_range);