Skip to content

Commit 2bc0771

Browse files
Hou Taogregkh
Hou Tao
authored andcommitted
virtiofs: use pages instead of pointer for kernel direct IO
[ Upstream commit 4174867 ] When trying to insert a 10MB kernel module kept in a virtio-fs with cache disabled, the following warning was reported: ------------[ cut here ]------------ WARNING: CPU: 1 PID: 404 at mm/page_alloc.c:4551 ...... Modules linked in: CPU: 1 PID: 404 Comm: insmod Not tainted 6.9.0-rc5+ Rust-for-Linux#123 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) ...... RIP: 0010:__alloc_pages+0x2bf/0x380 ...... Call Trace: <TASK> ? __warn+0x8e/0x150 ? __alloc_pages+0x2bf/0x380 __kmalloc_large_node+0x86/0x160 __kmalloc+0x33c/0x480 virtio_fs_enqueue_req+0x240/0x6d0 virtio_fs_wake_pending_and_unlock+0x7f/0x190 queue_request_and_unlock+0x55/0x60 fuse_simple_request+0x152/0x2b0 fuse_direct_io+0x5d2/0x8c0 fuse_file_read_iter+0x121/0x160 __kernel_read+0x151/0x2d0 kernel_read+0x45/0x50 kernel_read_file+0x1a9/0x2a0 init_module_from_file+0x6a/0xe0 idempotent_init_module+0x175/0x230 __x64_sys_finit_module+0x5d/0xb0 x64_sys_call+0x1c3/0x9e0 do_syscall_64+0x3d/0xc0 entry_SYSCALL_64_after_hwframe+0x4b/0x53 ...... </TASK> ---[ end trace 0000000000000000 ]--- The warning is triggered as follows: 1) syscall finit_module() handles the module insertion and it invokes kernel_read_file() to read the content of the module first. 2) kernel_read_file() allocates a 10MB buffer by using vmalloc() and passes it to kernel_read(). kernel_read() constructs a kvec iter by using iov_iter_kvec() and passes it to fuse_file_read_iter(). 3) virtio-fs disables the cache, so fuse_file_read_iter() invokes fuse_direct_io(). As for now, the maximal read size for kvec iter is only limited by fc->max_read. For virtio-fs, max_read is UINT_MAX, so fuse_direct_io() doesn't split the 10MB buffer. It saves the address and the size of the 10MB-sized buffer in out_args[0] of a fuse request and passes the fuse request to virtio_fs_wake_pending_and_unlock(). 4) virtio_fs_wake_pending_and_unlock() uses virtio_fs_enqueue_req() to queue the request. Because virtiofs need DMA-able address, so virtio_fs_enqueue_req() uses kmalloc() to allocate a bounce buffer for all fuse args, copies these args into the bounce buffer and passed the physical address of the bounce buffer to virtiofsd. The total length of these fuse args for the passed fuse request is about 10MB, so copy_args_to_argbuf() invokes kmalloc() with a 10MB size parameter and it triggers the warning in __alloc_pages(): if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)) return NULL; 5) virtio_fs_enqueue_req() will retry the memory allocation in a kworker, but it won't help, because kmalloc() will always return NULL due to the abnormal size and finit_module() will hang forever. A feasible solution is to limit the value of max_read for virtio-fs, so the length passed to kmalloc() will be limited. However it will affect the maximal read size for normal read. And for virtio-fs write initiated from kernel, it has the similar problem but now there is no way to limit fc->max_write in kernel. So instead of limiting both the values of max_read and max_write in kernel, introducing use_pages_for_kvec_io in fuse_conn and setting it as true in virtiofs. When use_pages_for_kvec_io is enabled, fuse will use pages instead of pointer to pass the KVEC_IO data. After switching to pages for KVEC_IO data, these pages will be used for DMA through virtio-fs. If these pages are backed by vmalloc(), {flush|invalidate}_kernel_vmap_range() are necessary to flush or invalidate the cache before the DMA operation. So add two new fields in fuse_args_pages to record the base address of vmalloc area and the condition indicating whether invalidation is needed. Perform the flush in fuse_get_user_pages() for write operations and the invalidation in fuse_release_user_pages() for read operations. It may seem necessary to introduce another field in fuse_conn to indicate that these KVEC_IO pages are used for DMA, However, considering that virtio-fs is currently the only user of use_pages_for_kvec_io, just reuse use_pages_for_kvec_io to indicate that these pages will be used for DMA. Fixes: a62a8ef ("virtio-fs: add virtiofs filesystem") Signed-off-by: Hou Tao <[email protected]> Tested-by: Jingbo Xu <[email protected]> Signed-off-by: Miklos Szeredi <[email protected]> Signed-off-by: Sasha Levin <[email protected]>
1 parent 4ad9592 commit 2bc0771

File tree

3 files changed

+50
-19
lines changed

3 files changed

+50
-19
lines changed

fs/fuse/file.c

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -645,7 +645,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
645645
args->out_args[0].size = count;
646646
}
647647

648-
static void fuse_release_user_pages(struct fuse_args_pages *ap,
648+
static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres,
649649
bool should_dirty)
650650
{
651651
unsigned int i;
@@ -656,6 +656,9 @@ static void fuse_release_user_pages(struct fuse_args_pages *ap,
656656
if (ap->args.is_pinned)
657657
unpin_user_page(ap->pages[i]);
658658
}
659+
660+
if (nres > 0 && ap->args.invalidate_vmap)
661+
invalidate_kernel_vmap_range(ap->args.vmap_base, nres);
659662
}
660663

661664
static void fuse_io_release(struct kref *kref)
@@ -754,25 +757,29 @@ static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
754757
struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
755758
struct fuse_io_priv *io = ia->io;
756759
ssize_t pos = -1;
757-
758-
fuse_release_user_pages(&ia->ap, io->should_dirty);
760+
size_t nres;
759761

760762
if (err) {
761763
/* Nothing */
762764
} else if (io->write) {
763765
if (ia->write.out.size > ia->write.in.size) {
764766
err = -EIO;
765-
} else if (ia->write.in.size != ia->write.out.size) {
766-
pos = ia->write.in.offset - io->offset +
767-
ia->write.out.size;
767+
} else {
768+
nres = ia->write.out.size;
769+
if (ia->write.in.size != ia->write.out.size)
770+
pos = ia->write.in.offset - io->offset +
771+
ia->write.out.size;
768772
}
769773
} else {
770774
u32 outsize = args->out_args[0].size;
771775

776+
nres = outsize;
772777
if (ia->read.in.size != outsize)
773778
pos = ia->read.in.offset - io->offset + outsize;
774779
}
775780

781+
fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty);
782+
776783
fuse_aio_complete(io, err, pos);
777784
fuse_io_free(ia);
778785
}
@@ -1468,24 +1475,37 @@ static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
14681475

14691476
static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
14701477
size_t *nbytesp, int write,
1471-
unsigned int max_pages)
1478+
unsigned int max_pages,
1479+
bool use_pages_for_kvec_io)
14721480
{
1481+
bool flush_or_invalidate = false;
14731482
size_t nbytes = 0; /* # bytes already packed in req */
14741483
ssize_t ret = 0;
14751484

1476-
/* Special case for kernel I/O: can copy directly into the buffer */
1485+
/* Special case for kernel I/O: can copy directly into the buffer.
1486+
* However if the implementation of fuse_conn requires pages instead of
1487+
* pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead.
1488+
*/
14771489
if (iov_iter_is_kvec(ii)) {
1478-
unsigned long user_addr = fuse_get_user_addr(ii);
1479-
size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1490+
void *user_addr = (void *)fuse_get_user_addr(ii);
14801491

1481-
if (write)
1482-
ap->args.in_args[1].value = (void *) user_addr;
1483-
else
1484-
ap->args.out_args[0].value = (void *) user_addr;
1492+
if (!use_pages_for_kvec_io) {
1493+
size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
14851494

1486-
iov_iter_advance(ii, frag_size);
1487-
*nbytesp = frag_size;
1488-
return 0;
1495+
if (write)
1496+
ap->args.in_args[1].value = user_addr;
1497+
else
1498+
ap->args.out_args[0].value = user_addr;
1499+
1500+
iov_iter_advance(ii, frag_size);
1501+
*nbytesp = frag_size;
1502+
return 0;
1503+
}
1504+
1505+
if (is_vmalloc_addr(user_addr)) {
1506+
ap->args.vmap_base = user_addr;
1507+
flush_or_invalidate = true;
1508+
}
14891509
}
14901510

14911511
while (nbytes < *nbytesp && ap->num_pages < max_pages) {
@@ -1514,6 +1534,10 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
15141534
(PAGE_SIZE - ret) & (PAGE_SIZE - 1);
15151535
}
15161536

1537+
if (write && flush_or_invalidate)
1538+
flush_kernel_vmap_range(ap->args.vmap_base, nbytes);
1539+
1540+
ap->args.invalidate_vmap = !write && flush_or_invalidate;
15171541
ap->args.is_pinned = iov_iter_extract_will_pin(ii);
15181542
ap->args.user_pages = true;
15191543
if (write)
@@ -1582,7 +1606,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
15821606
size_t nbytes = min(count, nmax);
15831607

15841608
err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1585-
max_pages);
1609+
max_pages, fc->use_pages_for_kvec_io);
15861610
if (err && !nbytes)
15871611
break;
15881612

@@ -1596,7 +1620,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
15961620
}
15971621

15981622
if (!io->async || nres < 0) {
1599-
fuse_release_user_pages(&ia->ap, io->should_dirty);
1623+
fuse_release_user_pages(&ia->ap, nres, io->should_dirty);
16001624
fuse_io_free(ia);
16011625
}
16021626
ia = NULL;

fs/fuse/fuse_i.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,9 +309,12 @@ struct fuse_args {
309309
bool may_block:1;
310310
bool is_ext:1;
311311
bool is_pinned:1;
312+
bool invalidate_vmap:1;
312313
struct fuse_in_arg in_args[3];
313314
struct fuse_arg out_args[2];
314315
void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
316+
/* Used for kvec iter backed by vmalloc address */
317+
void *vmap_base;
315318
};
316319

317320
struct fuse_args_pages {
@@ -857,6 +860,9 @@ struct fuse_conn {
857860
/** Passthrough support for read/write IO */
858861
unsigned int passthrough:1;
859862

863+
/* Use pages instead of pointer for kernel I/O */
864+
unsigned int use_pages_for_kvec_io:1;
865+
860866
/** Maximum stack depth for passthrough backing files */
861867
int max_stack_depth;
862868

fs/fuse/virtio_fs.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1691,6 +1691,7 @@ static int virtio_fs_get_tree(struct fs_context *fsc)
16911691
fc->delete_stale = true;
16921692
fc->auto_submounts = true;
16931693
fc->sync_fs = true;
1694+
fc->use_pages_for_kvec_io = true;
16941695

16951696
/* Tell FUSE to split requests that exceed the virtqueue's size */
16961697
fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit,

0 commit comments

Comments
 (0)