drivers/vhost/vhost.c

   1 /* Copyright (C) 2009 Red Hat, Inc.
   2  * Copyright (C) 2006 Rusty Russell IBM Corporation
   3  *
   4  * Author: Michael S. Tsirkin <mst@redhat.com>
   5  *
   6  * Inspiration, some code, and most witty comments come from
   7  * Documentation/virtual/lguest/lguest.c, by Rusty Russell
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.
  10  *
  11  * Generic code for virtio server in host kernel.
  12  */
  13
  14 #include <linux/eventfd.h>
  15 #include <linux/vhost.h>
  16 #include <linux/uio.h>
  17 #include <linux/mm.h>
  18 #include <linux/mmu_context.h>
  19 #include <linux/miscdevice.h>
  20 #include <linux/mutex.h>
  21 #include <linux/poll.h>
  22 #include <linux/file.h>
  23 #include <linux/highmem.h>
  24 #include <linux/slab.h>
  25 #include <linux/vmalloc.h>
  26 #include <linux/kthread.h>
  27 #include <linux/cgroup.h>
  28 #include <linux/module.h>
  29 #include <linux/sort.h>
  30 #include <linux/interval_tree_generic.h>
  31
  32 #include "vhost.h"
  33
  34 static ushort max_mem_regions = 64;
  35 module_param(max_mem_regions, ushort, 0444);
  36 MODULE_PARM_DESC(max_mem_regions,
  37         "Maximum number of memory regions in memory map. (default: 64)");
  38 static int max_iotlb_entries = 2048;
  39 module_param(max_iotlb_entries, int, 0444);
  40 MODULE_PARM_DESC(max_iotlb_entries,
  41         "Maximum number of iotlb entries. (default: 2048)");
  42
  43 enum {
  44         VHOST_MEMORY_F_LOG = 0x1,
  45 };
  46
  47 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
  48 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
  49
  50 INTERVAL_TREE_DEFINE(struct vhost_umem_node,
  51                      rb, __u64, __subtree_last,
  52                      START, LAST, , vhost_umem_interval_tree);
  53
  54 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
  55 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
  56 {
  57         vq->user_be = !virtio_legacy_is_little_endian();
  58 }
  59
  60 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
  61 {
  62         vq->user_be = true;
  63 }
  64
  65 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
  66 {
  67         vq->user_be = false;
  68 }
  69
  70 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
  71 {
  72         struct vhost_vring_state s;
  73
  74         if (vq->private_data)
  75                 return -EBUSY;
  76
  77         if (copy_from_user(&s, argp, sizeof(s)))
  78                 return -EFAULT;
  79
  80         if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
  81             s.num != VHOST_VRING_BIG_ENDIAN)
  82                 return -EINVAL;
  83
  84         if (s.num == VHOST_VRING_BIG_ENDIAN)
  85                 vhost_enable_cross_endian_big(vq);
  86         else
  87                 vhost_enable_cross_endian_little(vq);
  88
  89         return 0;
  90 }
  91
  92 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
  93                                    int __user *argp)
  94 {
  95         struct vhost_vring_state s = {
  96                 .index = idx,
  97                 .num = vq->user_be
  98         };
  99
 100         if (copy_to_user(argp, &s, sizeof(s)))
 101                 return -EFAULT;
 102
 103         return 0;
 104 }
 105
 106 static void vhost_init_is_le(struct vhost_virtqueue *vq)
 107 {
 108         /* Note for legacy virtio: user_be is initialized at reset time
 109          * according to the host endianness. If userspace does not set an
 110          * explicit endianness, the default behavior is native endian, as
 111          * expected by legacy virtio.
 112          */
 113         vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
 114 }
 115 #else
 116 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
 117 {
 118 }
 119
 120 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
 121 {
 122         return -ENOIOCTLCMD;
 123 }
 124
 125 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
 126                                    int __user *argp)
 127 {
 128         return -ENOIOCTLCMD;
 129 }
 130
 131 static void vhost_init_is_le(struct vhost_virtqueue *vq)
 132 {
 133         if (vhost_has_feature(vq, VIRTIO_F_VERSION_1))
 134                 vq->is_le = true;
 135 }
 136 #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
 137
 138 static void vhost_reset_is_le(struct vhost_virtqueue *vq)
 139 {
 140         vq->is_le = virtio_legacy_is_little_endian();
 141 }
 142
 143 struct vhost_flush_struct {
 144         struct vhost_work work;
 145         struct completion wait_event;
 146 };
 147
 148 static void vhost_flush_work(struct vhost_work *work)
 149 {
 150         struct vhost_flush_struct *s;
 151
 152         s = container_of(work, struct vhost_flush_struct, work);
 153         complete(&s->wait_event);
 154 }
 155
 156 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
 157                             poll_table *pt)
 158 {
 159         struct vhost_poll *poll;
 160
 161         poll = container_of(pt, struct vhost_poll, table);
 162         poll->wqh = wqh;
 163         add_wait_queue(wqh, &poll->wait);
 164 }
 165
 166 static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
 167                              void *key)
 168 {
 169         struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
 170
 171         if (!((unsigned long)key & poll->mask))
 172                 return 0;
 173
 174         vhost_poll_queue(poll);
 175         return 0;
 176 }
 177
 178 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
 179 {
 180         clear_bit(VHOST_WORK_QUEUED, &work->flags);
 181         work->fn = fn;
 182         init_waitqueue_head(&work->done);
 183 }
 184 EXPORT_SYMBOL_GPL(vhost_work_init);
 185
 186 /* Init poll structure */
 187 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
 188                      unsigned long mask, struct vhost_dev *dev)
 189 {
 190         init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
 191         init_poll_funcptr(&poll->table, vhost_poll_func);
 192         poll->mask = mask;
 193         poll->dev = dev;
 194         poll->wqh = NULL;
 195
 196         vhost_work_init(&poll->work, fn);
 197 }
 198 EXPORT_SYMBOL_GPL(vhost_poll_init);
 199
 200 /* Start polling a file. We add ourselves to file's wait queue. The caller must
 201  * keep a reference to a file until after vhost_poll_stop is called. */
 202 int vhost_poll_start(struct vhost_poll *poll, struct file *file)
 203 {
 204         unsigned long mask;
 205         int ret = 0;
 206
 207         if (poll->wqh)
 208                 return 0;
 209
 210         mask = file->f_op->poll(file, &poll->table);
 211         if (mask)
 212                 vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
 213         if (mask & POLLERR) {
 214                 if (poll->wqh)
 215                         remove_wait_queue(poll->wqh, &poll->wait);
 216                 ret = -EINVAL;
 217         }
 218
 219         return ret;
 220 }
 221 EXPORT_SYMBOL_GPL(vhost_poll_start);
 222
 223 /* Stop polling a file. After this function returns, it becomes safe to drop the
 224  * file reference. You must also flush afterwards. */
 225 void vhost_poll_stop(struct vhost_poll *poll)
 226 {
 227         if (poll->wqh) {
 228                 remove_wait_queue(poll->wqh, &poll->wait);
 229                 poll->wqh = NULL;
 230         }
 231 }
 232 EXPORT_SYMBOL_GPL(vhost_poll_stop);
 233
 234 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
 235 {
 236         struct vhost_flush_struct flush;
 237
 238         if (dev->worker) {
 239                 init_completion(&flush.wait_event);
 240                 vhost_work_init(&flush.work, vhost_flush_work);
 241
 242                 vhost_work_queue(dev, &flush.work);
 243                 wait_for_completion(&flush.wait_event);
 244         }
 245 }
 246 EXPORT_SYMBOL_GPL(vhost_work_flush);
 247
 248 /* Flush any work that has been scheduled. When calling this, don't hold any
 249  * locks that are also used by the callback. */
 250 void vhost_poll_flush(struct vhost_poll *poll)
 251 {
 252         vhost_work_flush(poll->dev, &poll->work);
 253 }
 254 EXPORT_SYMBOL_GPL(vhost_poll_flush);
 255
 256 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
 257 {
 258         if (!dev->worker)
 259                 return;
 260
 261         if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
 262                 /* We can only add the work to the list after we're
 263                  * sure it was not in the list.
 264                  */
 265                 smp_mb();
 266                 llist_add(&work->node, &dev->work_list);
 267                 wake_up_process(dev->worker);
 268         }
 269 }
 270 EXPORT_SYMBOL_GPL(vhost_work_queue);
 271
 272 /* A lockless hint for busy polling code to exit the loop */
 273 bool vhost_has_work(struct vhost_dev *dev)
 274 {
 275         return !llist_empty(&dev->work_list);
 276 }
 277 EXPORT_SYMBOL_GPL(vhost_has_work);
 278
 279 void vhost_poll_queue(struct vhost_poll *poll)
 280 {
 281         vhost_work_queue(poll->dev, &poll->work);
 282 }
 283 EXPORT_SYMBOL_GPL(vhost_poll_queue);
 284
 285 static void vhost_vq_reset(struct vhost_dev *dev,
 286                            struct vhost_virtqueue *vq)
 287 {
 288         vq->num = 1;
 289         vq->desc = NULL;
 290         vq->avail = NULL;
 291         vq->used = NULL;
 292         vq->last_avail_idx = 0;
 293         vq->avail_idx = 0;
 294         vq->last_used_idx = 0;
 295         vq->signalled_used = 0;
 296         vq->signalled_used_valid = false;
 297         vq->used_flags = 0;
 298         vq->log_used = false;
 299         vq->log_addr = -1ull;
 300         vq->private_data = NULL;
 301         vq->acked_features = 0;
 302         vq->log_base = NULL;
 303         vq->error_ctx = NULL;
 304         vq->error = NULL;
 305         vq->kick = NULL;
 306         vq->call_ctx = NULL;
 307         vq->call = NULL;
 308         vq->log_ctx = NULL;
 309         vhost_reset_is_le(vq);
 310         vhost_disable_cross_endian(vq);
 311         vq->busyloop_timeout = 0;
 312         vq->umem = NULL;
 313         vq->iotlb = NULL;
 314 }
 315
 316 static int vhost_worker(void *data)
 317 {
 318         struct vhost_dev *dev = data;
 319         struct vhost_work *work, *work_next;
 320         struct llist_node *node;
 321         mm_segment_t oldfs = get_fs();
 322
 323         set_fs(USER_DS);
 324         use_mm(dev->mm);
 325
 326         for (;;) {
 327                 /* mb paired w/ kthread_stop */
 328                 set_current_state(TASK_INTERRUPTIBLE);
 329
 330                 if (kthread_should_stop()) {
 331                         __set_current_state(TASK_RUNNING);
 332                         break;
 333                 }
 334
 335                 node = llist_del_all(&dev->work_list);
 336                 if (!node)
 337                         schedule();
 338
 339                 node = llist_reverse_order(node);
 340                 /* make sure flag is seen after deletion */
 341                 smp_wmb();
 342                 llist_for_each_entry_safe(work, work_next, node, node) {
 343                         clear_bit(VHOST_WORK_QUEUED, &work->flags);
 344                         __set_current_state(TASK_RUNNING);
 345                         work->fn(work);
 346                         if (need_resched())
 347                                 schedule();
 348                 }
 349         }
 350         unuse_mm(dev->mm);
 351         set_fs(oldfs);
 352         return 0;
 353 }
 354
 355 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
 356 {
 357         kfree(vq->indirect);
 358         vq->indirect = NULL;
 359         kfree(vq->log);
 360         vq->log = NULL;
 361         kfree(vq->heads);
 362         vq->heads = NULL;
 363 }
 364
 365 /* Helper to allocate iovec buffers for all vqs. */
 366 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
 367 {
 368         struct vhost_virtqueue *vq;
 369         int i;
 370
 371         for (i = 0; i < dev->nvqs; ++i) {
 372                 vq = dev->vqs[i];
 373                 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
 374                                        GFP_KERNEL);
 375                 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
 376                 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
 377                 if (!vq->indirect || !vq->log || !vq->heads)
 378                         goto err_nomem;
 379         }
 380         return 0;
 381
 382 err_nomem:
 383         for (; i >= 0; --i)
 384                 vhost_vq_free_iovecs(dev->vqs[i]);
 385         return -ENOMEM;
 386 }
 387
 388 static void vhost_dev_free_iovecs(struct vhost_dev *dev)
 389 {
 390         int i;
 391
 392         for (i = 0; i < dev->nvqs; ++i)
 393                 vhost_vq_free_iovecs(dev->vqs[i]);
 394 }
 395
 396 void vhost_dev_init(struct vhost_dev *dev,
 397                     struct vhost_virtqueue **vqs, int nvqs)
 398 {
 399         struct vhost_virtqueue *vq;
 400         int i;
 401
 402         dev->vqs = vqs;
 403         dev->nvqs = nvqs;
 404         mutex_init(&dev->mutex);
 405         dev->log_ctx = NULL;
 406         dev->log_file = NULL;
 407         dev->umem = NULL;
 408         dev->iotlb = NULL;
 409         dev->mm = NULL;
 410         dev->worker = NULL;
 411         init_llist_head(&dev->work_list);
 412         init_waitqueue_head(&dev->wait);
 413         INIT_LIST_HEAD(&dev->read_list);
 414         INIT_LIST_HEAD(&dev->pending_list);
 415         spin_lock_init(&dev->iotlb_lock);
 416
 417
 418         for (i = 0; i < dev->nvqs; ++i) {
 419                 vq = dev->vqs[i];
 420                 vq->log = NULL;
 421                 vq->indirect = NULL;
 422                 vq->heads = NULL;
 423                 vq->dev = dev;
 424                 mutex_init(&vq->mutex);
 425                 vhost_vq_reset(dev, vq);
 426                 if (vq->handle_kick)
 427                         vhost_poll_init(&vq->poll, vq->handle_kick,
 428                                         POLLIN, dev);
 429         }
 430 }
 431 EXPORT_SYMBOL_GPL(vhost_dev_init);
 432
 433 /* Caller should have device mutex */
 434 long vhost_dev_check_owner(struct vhost_dev *dev)
 435 {
 436         /* Are you the owner? If not, I don't think you mean to do that */
 437         return dev->mm == current->mm ? 0 : -EPERM;
 438 }
 439 EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
 440
 441 struct vhost_attach_cgroups_struct {
 442         struct vhost_work work;
 443         struct task_struct *owner;
 444         int ret;
 445 };
 446
 447 static void vhost_attach_cgroups_work(struct vhost_work *work)
 448 {
 449         struct vhost_attach_cgroups_struct *s;
 450
 451         s = container_of(work, struct vhost_attach_cgroups_struct, work);
 452         s->ret = cgroup_attach_task_all(s->owner, current);
 453 }
 454
 455 static int vhost_attach_cgroups(struct vhost_dev *dev)
 456 {
 457         struct vhost_attach_cgroups_struct attach;
 458
 459         attach.owner = current;
 460         vhost_work_init(&attach.work, vhost_attach_cgroups_work);
 461         vhost_work_queue(dev, &attach.work);
 462         vhost_work_flush(dev, &attach.work);
 463         return attach.ret;
 464 }
 465
 466 /* Caller should have device mutex */
 467 bool vhost_dev_has_owner(struct vhost_dev *dev)
 468 {
 469         return dev->mm;
 470 }
 471 EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
 472
 473 /* Caller should have device mutex */
 474 long vhost_dev_set_owner(struct vhost_dev *dev)
 475 {
 476         struct task_struct *worker;
 477         int err;
 478
 479         /* Is there an owner already? */
 480         if (vhost_dev_has_owner(dev)) {
 481                 err = -EBUSY;
 482                 goto err_mm;
 483         }
 484
 485         /* No owner, become one */
 486         dev->mm = get_task_mm(current);
 487         worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
 488         if (IS_ERR(worker)) {
 489                 err = PTR_ERR(worker);
 490                 goto err_worker;
 491         }
 492
 493         dev->worker = worker;
 494         wake_up_process(worker);        /* avoid contributing to loadavg */
 495
 496         err = vhost_attach_cgroups(dev);
 497         if (err)
 498                 goto err_cgroup;
 499
 500         err = vhost_dev_alloc_iovecs(dev);
 501         if (err)
 502                 goto err_cgroup;
 503
 504         return 0;
 505 err_cgroup:
 506         kthread_stop(worker);
 507         dev->worker = NULL;
 508 err_worker:
 509         if (dev->mm)
 510                 mmput(dev->mm);
 511         dev->mm = NULL;
 512 err_mm:
 513         return err;
 514 }
 515 EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
 516
 517 static void *vhost_kvzalloc(unsigned long size)
 518 {
 519         void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
 520
 521         if (!n)
 522                 n = vzalloc(size);
 523         return n;
 524 }
 525
 526 struct vhost_umem *vhost_dev_reset_owner_prepare(void)
 527 {
 528         return vhost_kvzalloc(sizeof(struct vhost_umem));
 529 }
 530 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
 531
 532 /* Caller should have device mutex */
 533 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
 534 {
 535         int i;
 536
 537         vhost_dev_cleanup(dev, true);
 538
 539         /* Restore memory to default empty mapping. */
 540         INIT_LIST_HEAD(&umem->umem_list);
 541         dev->umem = umem;
 542         /* We don't need VQ locks below since vhost_dev_cleanup makes sure
 543          * VQs aren't running.
 544          */
 545         for (i = 0; i < dev->nvqs; ++i)
 546                 dev->vqs[i]->umem = umem;
 547 }
 548 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
 549
 550 void vhost_dev_stop(struct vhost_dev *dev)
 551 {
 552         int i;
 553
 554         for (i = 0; i < dev->nvqs; ++i) {
 555                 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
 556                         vhost_poll_stop(&dev->vqs[i]->poll);
 557                         vhost_poll_flush(&dev->vqs[i]->poll);
 558                 }
 559         }
 560 }
 561 EXPORT_SYMBOL_GPL(vhost_dev_stop);
 562
 563 static void vhost_umem_free(struct vhost_umem *umem,
 564                             struct vhost_umem_node *node)
 565 {
 566         vhost_umem_interval_tree_remove(node, &umem->umem_tree);
 567         list_del(&node->link);
 568         kfree(node);
 569         umem->numem--;
 570 }
 571
 572 static void vhost_umem_clean(struct vhost_umem *umem)
 573 {
 574         struct vhost_umem_node *node, *tmp;
 575
 576         if (!umem)
 577                 return;
 578
 579         list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
 580                 vhost_umem_free(umem, node);
 581
 582         kvfree(umem);
 583 }
 584
 585 static void vhost_clear_msg(struct vhost_dev *dev)
 586 {
 587         struct vhost_msg_node *node, *n;
 588
 589         spin_lock(&dev->iotlb_lock);
 590
 591         list_for_each_entry_safe(node, n, &dev->read_list, node) {
 592                 list_del(&node->node);
 593                 kfree(node);
 594         }
 595
 596         list_for_each_entry_safe(node, n, &dev->pending_list, node) {
 597                 list_del(&node->node);
 598                 kfree(node);
 599         }
 600
 601         spin_unlock(&dev->iotlb_lock);
 602 }
 603
 604 /* Caller should have device mutex if and only if locked is set */
 605 void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
 606 {
 607         int i;
 608
 609         for (i = 0; i < dev->nvqs; ++i) {
 610                 if (dev->vqs[i]->error_ctx)
 611                         eventfd_ctx_put(dev->vqs[i]->error_ctx);
 612                 if (dev->vqs[i]->error)
 613                         fput(dev->vqs[i]->error);
 614                 if (dev->vqs[i]->kick)
 615                         fput(dev->vqs[i]->kick);
 616                 if (dev->vqs[i]->call_ctx)
 617                         eventfd_ctx_put(dev->vqs[i]->call_ctx);
 618                 if (dev->vqs[i]->call)
 619                         fput(dev->vqs[i]->call);
 620                 vhost_vq_reset(dev, dev->vqs[i]);
 621         }
 622         vhost_dev_free_iovecs(dev);
 623         if (dev->log_ctx)
 624                 eventfd_ctx_put(dev->log_ctx);
 625         dev->log_ctx = NULL;
 626         if (dev->log_file)
 627                 fput(dev->log_file);
 628         dev->log_file = NULL;
 629         /* No one will access memory at this point */
 630         vhost_umem_clean(dev->umem);
 631         dev->umem = NULL;
 632         vhost_umem_clean(dev->iotlb);
 633         dev->iotlb = NULL;
 634         vhost_clear_msg(dev);
 635         wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM);
 636         WARN_ON(!llist_empty(&dev->work_list));
 637         if (dev->worker) {
 638                 kthread_stop(dev->worker);
 639                 dev->worker = NULL;
 640         }
 641         if (dev->mm)
 642                 mmput(dev->mm);
 643         dev->mm = NULL;
 644 }
 645 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
 646
 647 static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
 648 {
 649         u64 a = addr / VHOST_PAGE_SIZE / 8;
 650
 651         /* Make sure 64 bit math will not overflow. */
 652         if (a > ULONG_MAX - (unsigned long)log_base ||
 653             a + (unsigned long)log_base > ULONG_MAX)
 654                 return 0;
 655
 656         return access_ok(VERIFY_WRITE, log_base + a,
 657                          (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
 658 }
 659
 660 /* Caller should have vq mutex and device mutex. */
 661 static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
 662                                int log_all)
 663 {
 664         struct vhost_umem_node *node;
 665
 666         if (!umem)
 667                 return 0;
 668
 669         list_for_each_entry(node, &umem->umem_list, link) {
 670                 unsigned long a = node->userspace_addr;
 671
 672                 if (node->size > ULONG_MAX)
 673                         return 0;
 674                 else if (!access_ok(VERIFY_WRITE, (void __user *)a,
 675                                     node->size))
 676                         return 0;
 677                 else if (log_all && !log_access_ok(log_base,
 678                                                    node->start,
 679                                                    node->size))
 680                         return 0;
 681         }
 682         return 1;
 683 }
 684
 685 /* Can we switch to this memory table? */
 686 /* Caller should have device mutex but not vq mutex */
 687 static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
 688                             int log_all)
 689 {
 690         int i;
 691
 692         for (i = 0; i < d->nvqs; ++i) {
 693                 int ok;
 694                 bool log;
 695
 696                 mutex_lock(&d->vqs[i]->mutex);
 697                 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
 698                 /* If ring is inactive, will check when it's enabled. */
 699                 if (d->vqs[i]->private_data)
 700                         ok = vq_memory_access_ok(d->vqs[i]->log_base,
 701                                                  umem, log);
 702                 else
 703                         ok = 1;
 704                 mutex_unlock(&d->vqs[i]->mutex);
 705                 if (!ok)
 706                         return 0;
 707         }
 708         return 1;
 709 }
 710
 711 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
 712                           struct iovec iov[], int iov_size, int access);
 713
 714 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void *to,
 715                               const void *from, unsigned size)
 716 {
 717         int ret;
 718
 719         if (!vq->iotlb)
 720                 return __copy_to_user(to, from, size);
 721         else {
 722                 /* This function should be called after iotlb
 723                  * prefetch, which means we're sure that all vq
 724                  * could be access through iotlb. So -EAGAIN should
 725                  * not happen in this case.
 726                  */
 727                 /* TODO: more fast path */
 728                 struct iov_iter t;
 729                 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
 730                                      ARRAY_SIZE(vq->iotlb_iov),
 731                                      VHOST_ACCESS_WO);
 732                 if (ret < 0)
 733                         goto out;
 734                 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size);
 735                 ret = copy_to_iter(from, size, &t);
 736                 if (ret == size)
 737                         ret = 0;
 738         }
 739 out:
 740         return ret;
 741 }
 742
 743 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
 744                                 void *from, unsigned size)
 745 {
 746         int ret;
 747
 748         if (!vq->iotlb)
 749                 return __copy_from_user(to, from, size);
 750         else {
 751                 /* This function should be called after iotlb
 752                  * prefetch, which means we're sure that vq
 753                  * could be access through iotlb. So -EAGAIN should
 754                  * not happen in this case.
 755                  */
 756                 /* TODO: more fast path */
 757                 struct iov_iter f;
 758                 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
 759                                      ARRAY_SIZE(vq->iotlb_iov),
 760                                      VHOST_ACCESS_RO);
 761                 if (ret < 0) {
 762                         vq_err(vq, "IOTLB translation failure: uaddr "
 763                                "%p size 0x%llx\n", from,
 764                                (unsigned long long) size);
 765                         goto out;
 766                 }
 767                 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size);
 768                 ret = copy_from_iter(to, size, &f);
 769                 if (ret == size)
 770                         ret = 0;
 771         }
 772
 773 out:
 774         return ret;
 775 }
 776
 777 static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
 778                                      void *addr, unsigned size)
 779 {
 780         int ret;
 781
 782         /* This function should be called after iotlb
 783          * prefetch, which means we're sure that vq
 784          * could be access through iotlb. So -EAGAIN should
 785          * not happen in this case.
 786          */
 787         /* TODO: more fast path */
 788         ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
 789                              ARRAY_SIZE(vq->iotlb_iov),
 790                              VHOST_ACCESS_RO);
 791         if (ret < 0) {
 792                 vq_err(vq, "IOTLB translation failure: uaddr "
 793                         "%p size 0x%llx\n", addr,
 794                         (unsigned long long) size);
 795                 return NULL;
 796         }
 797
 798         if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
 799                 vq_err(vq, "Non atomic userspace memory access: uaddr "
 800                         "%p size 0x%llx\n", addr,
 801                         (unsigned long long) size);
 802                 return NULL;
 803         }
 804
 805         return vq->iotlb_iov[0].iov_base;
 806 }
 807
 808 #define vhost_put_user(vq, x, ptr) \
 809 ({ \
 810         int ret = -EFAULT; \
 811         if (!vq->iotlb) { \
 812                 ret = __put_user(x, ptr); \
 813         } else { \
 814                 __typeof__(ptr) to = \
 815                         (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
 816                 if (to != NULL) \
 817                         ret = __put_user(x, to); \
 818                 else \
 819                         ret = -EFAULT;  \
 820         } \
 821         ret; \
 822 })
 823
 824 #define vhost_get_user(vq, x, ptr) \
 825 ({ \
 826         int ret; \
 827         if (!vq->iotlb) { \
 828                 ret = __get_user(x, ptr); \
 829         } else { \
 830                 __typeof__(ptr) from = \
 831                         (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
 832                 if (from != NULL) \
 833                         ret = __get_user(x, from); \
 834                 else \
 835                         ret = -EFAULT; \
 836         } \
 837         ret; \
 838 })
 839
 840 static void vhost_dev_lock_vqs(struct vhost_dev *d)
 841 {
 842         int i = 0;
 843         for (i = 0; i < d->nvqs; ++i)
 844                 mutex_lock(&d->vqs[i]->mutex);
 845 }
 846
 847 static void vhost_dev_unlock_vqs(struct vhost_dev *d)
 848 {
 849         int i = 0;
 850         for (i = 0; i < d->nvqs; ++i)
 851                 mutex_unlock(&d->vqs[i]->mutex);
 852 }
 853
 854 static int vhost_new_umem_range(struct vhost_umem *umem,
 855                                 u64 start, u64 size, u64 end,
 856                                 u64 userspace_addr, int perm)
 857 {
 858         struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC);
 859
 860         if (!node)
 861                 return -ENOMEM;
 862
 863         if (umem->numem == max_iotlb_entries) {
 864                 tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
 865                 vhost_umem_free(umem, tmp);
 866         }
 867
 868         node->start = start;
 869         node->size = size;
 870         node->last = end;
 871         node->userspace_addr = userspace_addr;
 872         node->perm = perm;
 873         INIT_LIST_HEAD(&node->link);
 874         list_add_tail(&node->link, &umem->umem_list);
 875         vhost_umem_interval_tree_insert(node, &umem->umem_tree);
 876         umem->numem++;
 877
 878         return 0;
 879 }
 880
 881 static void vhost_del_umem_range(struct vhost_umem *umem,
 882                                  u64 start, u64 end)
 883 {
 884         struct vhost_umem_node *node;
 885
 886         while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
 887                                                            start, end)))
 888                 vhost_umem_free(umem, node);
 889 }
 890
 891 static void vhost_iotlb_notify_vq(struct vhost_dev *d,
 892                                   struct vhost_iotlb_msg *msg)
 893 {
 894         struct vhost_msg_node *node, *n;
 895
 896         spin_lock(&d->iotlb_lock);
 897
 898         list_for_each_entry_safe(node, n, &d->pending_list, node) {
 899                 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
 900                 if (msg->iova <= vq_msg->iova &&
 901                     msg->iova + msg->size - 1 > vq_msg->iova &&
 902                     vq_msg->type == VHOST_IOTLB_MISS) {
 903                         vhost_poll_queue(&node->vq->poll);
 904                         list_del(&node->node);
 905                         kfree(node);
 906                 }
 907         }
 908
 909         spin_unlock(&d->iotlb_lock);
 910 }
 911
 912 static int umem_access_ok(u64 uaddr, u64 size, int access)
 913 {
 914         unsigned long a = uaddr;
 915
 916         if ((access & VHOST_ACCESS_RO) &&
 917             !access_ok(VERIFY_READ, (void __user *)a, size))
 918                 return -EFAULT;
 919         if ((access & VHOST_ACCESS_WO) &&
 920             !access_ok(VERIFY_WRITE, (void __user *)a, size))
 921                 return -EFAULT;
 922         return 0;
 923 }
 924
 925 int vhost_process_iotlb_msg(struct vhost_dev *dev,
 926                             struct vhost_iotlb_msg *msg)
 927 {
 928         int ret = 0;
 929
 930         vhost_dev_lock_vqs(dev);
 931         switch (msg->type) {
 932         case VHOST_IOTLB_UPDATE:
 933                 if (!dev->iotlb) {
 934                         ret = -EFAULT;
 935                         break;
 936                 }
 937                 if (umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
 938                         ret = -EFAULT;
 939                         break;
 940                 }
 941                 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
 942                                          msg->iova + msg->size - 1,
 943                                          msg->uaddr, msg->perm)) {
 944                         ret = -ENOMEM;
 945                         break;
 946                 }
 947                 vhost_iotlb_notify_vq(dev, msg);
 948                 break;
 949         case VHOST_IOTLB_INVALIDATE:
 950                 vhost_del_umem_range(dev->iotlb, msg->iova,
 951                                      msg->iova + msg->size - 1);
 952                 break;
 953         default:
 954                 ret = -EINVAL;
 955                 break;
 956         }
 957
 958         vhost_dev_unlock_vqs(dev);
 959         return ret;
 960 }
 961 ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
 962                              struct iov_iter *from)
 963 {
 964         struct vhost_msg_node node;
 965         unsigned size = sizeof(struct vhost_msg);
 966         size_t ret;
 967         int err;
 968
 969         if (iov_iter_count(from) < size)
 970                 return 0;
 971         ret = copy_from_iter(&node.msg, size, from);
 972         if (ret != size)
 973                 goto done;
 974
 975         switch (node.msg.type) {
 976         case VHOST_IOTLB_MSG:
 977                 err = vhost_process_iotlb_msg(dev, &node.msg.iotlb);
 978                 if (err)
 979                         ret = err;
 980                 break;
 981         default:
 982                 ret = -EINVAL;
 983                 break;
 984         }
 985
 986 done:
 987         return ret;
 988 }
 989 EXPORT_SYMBOL(vhost_chr_write_iter);
 990
 991 unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev,
 992                             poll_table *wait)
 993 {
 994         unsigned int mask = 0;
 995
 996         poll_wait(file, &dev->wait, wait);
 997
 998         if (!list_empty(&dev->read_list))
 999                 mask |= POLLIN | POLLRDNORM;
1000
1001         return mask;
1002 }
1003 EXPORT_SYMBOL(vhost_chr_poll);
1004
1005 ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
1006                             int noblock)
1007 {
1008         DEFINE_WAIT(wait);
1009         struct vhost_msg_node *node;
1010         ssize_t ret = 0;
1011         unsigned size = sizeof(struct vhost_msg);
1012
1013         if (iov_iter_count(to) < size)
1014                 return 0;
1015
1016         while (1) {
1017                 if (!noblock)
1018                         prepare_to_wait(&dev->wait, &wait,
1019                                         TASK_INTERRUPTIBLE);
1020
1021                 node = vhost_dequeue_msg(dev, &dev->read_list);
1022                 if (node)
1023                         break;
1024                 if (noblock) {
1025                         ret = -EAGAIN;
1026                         break;
1027                 }
1028                 if (signal_pending(current)) {
1029                         ret = -ERESTARTSYS;
1030                         break;
1031                 }
1032                 if (!dev->iotlb) {
1033                         ret = -EBADFD;
1034                         break;
1035                 }
1036
1037                 schedule();
1038         }
1039
1040         if (!noblock)
1041                 finish_wait(&dev->wait, &wait);
1042
1043         if (node) {
1044                 ret = copy_to_iter(&node->msg, size, to);
1045
1046                 if (ret != size || node->msg.type != VHOST_IOTLB_MISS) {
1047                         kfree(node);
1048                         return ret;
1049                 }
1050
1051                 vhost_enqueue_msg(dev, &dev->pending_list, node);
1052         }
1053
1054         return ret;
1055 }
1056 EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
1057
1058 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
1059 {
1060         struct vhost_dev *dev = vq->dev;
1061         struct vhost_msg_node *node;
1062         struct vhost_iotlb_msg *msg;
1063
1064         node = vhost_new_msg(vq, VHOST_IOTLB_MISS);
1065         if (!node)
1066                 return -ENOMEM;
1067
1068         msg = &node->msg.iotlb;
1069         msg->type = VHOST_IOTLB_MISS;
1070         msg->iova = iova;
1071         msg->perm = access;
1072
1073         vhost_enqueue_msg(dev, &dev->read_list, node);
1074
1075         return 0;
1076 }
1077
1078 static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1079                         struct vring_desc __user *desc,
1080                         struct vring_avail __user *avail,
1081                         struct vring_used __user *used)
1082
1083 {
1084         size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1085
1086         return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
1087                access_ok(VERIFY_READ, avail,
1088                          sizeof *avail + num * sizeof *avail->ring + s) &&
1089                access_ok(VERIFY_WRITE, used,
1090                         sizeof *used + num * sizeof *used->ring + s);
1091 }
1092
1093 static int iotlb_access_ok(struct vhost_virtqueue *vq,
1094                            int access, u64 addr, u64 len)
1095 {
1096         const struct vhost_umem_node *node;
1097         struct vhost_umem *umem = vq->iotlb;
1098         u64 s = 0, size;
1099
1100         while (len > s) {
1101                 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1102                                                            addr,
1103                                                            addr + len - 1);
1104                 if (node == NULL || node->start > addr) {
1105                         vhost_iotlb_miss(vq, addr, access);
1106                         return false;
1107                 } else if (!(node->perm & access)) {
1108                         /* Report the possible access violation by
1109                          * request another translation from userspace.
1110                          */
1111                         return false;
1112                 }
1113
1114                 size = node->size - addr + node->start;
1115                 s += size;
1116                 addr += size;
1117         }
1118
1119         return true;
1120 }
1121
1122 int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
1123 {
1124         size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1125         unsigned int num = vq->num;
1126
1127         if (!vq->iotlb)
1128                 return 1;
1129
1130         return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
1131                                num * sizeof *vq->desc) &&
1132                iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
1133                                sizeof *vq->avail +
1134                                num * sizeof *vq->avail->ring + s) &&
1135                iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
1136                                sizeof *vq->used +
1137                                num * sizeof *vq->used->ring + s);
1138 }
1139 EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);
1140
1141 /* Can we log writes? */
1142 /* Caller should have device mutex but not vq mutex */
1143 int vhost_log_access_ok(struct vhost_dev *dev)
1144 {
1145         return memory_access_ok(dev, dev->umem, 1);
1146 }
1147 EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1148
1149 /* Verify access for write logging. */
1150 /* Caller should have vq mutex and device mutex */
1151 static int vq_log_access_ok(struct vhost_virtqueue *vq,
1152                             void __user *log_base)
1153 {
1154         size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1155
1156         return vq_memory_access_ok(log_base, vq->umem,
1157                                    vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1158                 (!vq->log_used || log_access_ok(log_base, vq->log_addr,
1159                                         sizeof *vq->used +
1160                                         vq->num * sizeof *vq->used->ring + s));
1161 }
1162
1163 /* Can we start vq? */
1164 /* Caller should have vq mutex and device mutex */
1165 int vhost_vq_access_ok(struct vhost_virtqueue *vq)
1166 {
1167         if (vq->iotlb) {
1168                 /* When device IOTLB was used, the access validation
1169                  * will be validated during prefetching.
1170                  */
1171                 return 1;
1172         }
1173         return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) &&
1174                 vq_log_access_ok(vq, vq->log_base);
1175 }
1176 EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1177
1178 static struct vhost_umem *vhost_umem_alloc(void)
1179 {
1180         struct vhost_umem *umem = vhost_kvzalloc(sizeof(*umem));
1181
1182         if (!umem)
1183                 return NULL;
1184
1185         umem->umem_tree = RB_ROOT;
1186         umem->numem = 0;
1187         INIT_LIST_HEAD(&umem->umem_list);
1188
1189         return umem;
1190 }
1191
1192 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
1193 {
1194         struct vhost_memory mem, *newmem;
1195         struct vhost_memory_region *region;
1196         struct vhost_umem *newumem, *oldumem;
1197         unsigned long size = offsetof(struct vhost_memory, regions);
1198         int i;
1199
1200         if (copy_from_user(&mem, m, size))
1201                 return -EFAULT;
1202         if (mem.padding)
1203                 return -EOPNOTSUPP;
1204         if (mem.nregions > max_mem_regions)
1205                 return -E2BIG;
1206         newmem = vhost_kvzalloc(size + mem.nregions * sizeof(*m->regions));
1207         if (!newmem)
1208                 return -ENOMEM;
1209
1210         memcpy(newmem, &mem, size);
1211         if (copy_from_user(newmem->regions, m->regions,
1212                            mem.nregions * sizeof *m->regions)) {
1213                 kvfree(newmem);
1214                 return -EFAULT;
1215         }
1216
1217         newumem = vhost_umem_alloc();
1218         if (!newumem) {
1219                 kvfree(newmem);
1220                 return -ENOMEM;
1221         }
1222
1223         for (region = newmem->regions;
1224              region < newmem->regions + mem.nregions;
1225              region++) {
1226                 if (vhost_new_umem_range(newumem,
1227                                          region->guest_phys_addr,
1228                                          region->memory_size,
1229                                          region->guest_phys_addr +
1230                                          region->memory_size - 1,
1231                                          region->userspace_addr,
1232                                          VHOST_ACCESS_RW))
1233                         goto err;
1234         }
1235
1236         if (!memory_access_ok(d, newumem, 0))
1237                 goto err;
1238
1239         oldumem = d->umem;
1240         d->umem = newumem;
1241
1242         /* All memory accesses are done under some VQ mutex. */
1243         for (i = 0; i < d->nvqs; ++i) {
1244                 mutex_lock(&d->vqs[i]->mutex);
1245                 d->vqs[i]->umem = newumem;
1246                 mutex_unlock(&d->vqs[i]->mutex);
1247         }
1248
1249         kvfree(newmem);
1250         vhost_umem_clean(oldumem);
1251         return 0;
1252
1253 err:
1254         vhost_umem_clean(newumem);
1255         kvfree(newmem);
1256         return -EFAULT;
1257 }
1258
1259 long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
1260 {
1261         struct file *eventfp, *filep = NULL;
1262         bool pollstart = false, pollstop = false;
1263         struct eventfd_ctx *ctx = NULL;
1264         u32 __user *idxp = argp;
1265         struct vhost_virtqueue *vq;
1266         struct vhost_vring_state s;
1267         struct vhost_vring_file f;
1268         struct vhost_vring_addr a;
1269         u32 idx;
1270         long r;
1271
1272         r = get_user(idx, idxp);
1273         if (r < 0)
1274                 return r;
1275         if (idx >= d->nvqs)
1276                 return -ENOBUFS;
1277
1278         vq = d->vqs[idx];
1279
1280         mutex_lock(&vq->mutex);
1281
1282         switch (ioctl) {
1283         case VHOST_SET_VRING_NUM:
1284                 /* Resizing ring with an active backend?
1285                  * You don't want to do that. */
1286                 if (vq->private_data) {
1287                         r = -EBUSY;
1288                         break;
1289                 }
1290                 if (copy_from_user(&s, argp, sizeof s)) {
1291                         r = -EFAULT;
1292                         break;
1293                 }
1294                 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
1295                         r = -EINVAL;
1296                         break;
1297                 }
1298                 vq->num = s.num;
1299                 break;
1300         case VHOST_SET_VRING_BASE:
1301                 /* Moving base with an active backend?
1302                  * You don't want to do that. */
1303                 if (vq->private_data) {
1304                         r = -EBUSY;
1305                         break;
1306                 }
1307                 if (copy_from_user(&s, argp, sizeof s)) {
1308                         r = -EFAULT;
1309                         break;
1310                 }
1311                 if (s.num > 0xffff) {
1312                         r = -EINVAL;
1313                         break;
1314                 }
1315                 vq->last_avail_idx = s.num;
1316                 /* Forget the cached index value. */
1317                 vq->avail_idx = vq->last_avail_idx;
1318                 break;
1319         case VHOST_GET_VRING_BASE:
1320                 s.index = idx;
1321                 s.num = vq->last_avail_idx;
1322                 if (copy_to_user(argp, &s, sizeof s))
1323                         r = -EFAULT;
1324                 break;
1325         case VHOST_SET_VRING_ADDR:
1326                 if (copy_from_user(&a, argp, sizeof a)) {
1327                         r = -EFAULT;
1328                         break;
1329                 }
1330                 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
1331                         r = -EOPNOTSUPP;
1332                         break;
1333                 }
1334                 /* For 32bit, verify that the top 32bits of the user
1335                    data are set to zero. */
1336                 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
1337                     (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
1338                     (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
1339                         r = -EFAULT;
1340                         break;
1341                 }
1342
1343                 /* Make sure it's safe to cast pointers to vring types. */
1344                 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
1345                 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
1346                 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
1347                     (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1348                     (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
1349                         r = -EINVAL;
1350                         break;
1351                 }
1352
1353                 /* We only verify access here if backend is configured.
1354                  * If it is not, we don't as size might not have been setup.
1355                  * We will verify when backend is configured. */
1356                 if (vq->private_data) {
1357                         if (!vq_access_ok(vq, vq->num,
1358                                 (void __user *)(unsigned long)a.desc_user_addr,
1359                                 (void __user *)(unsigned long)a.avail_user_addr,
1360                                 (void __user *)(unsigned long)a.used_user_addr)) {
1361                                 r = -EINVAL;
1362                                 break;
1363                         }
1364
1365                         /* Also validate log access for used ring if enabled. */
1366                         if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
1367                             !log_access_ok(vq->log_base, a.log_guest_addr,
1368                                            sizeof *vq->used +
1369                                            vq->num * sizeof *vq->used->ring)) {
1370                                 r = -EINVAL;
1371                                 break;
1372                         }
1373                 }
1374
1375                 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
1376                 vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
1377                 vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
1378                 vq->log_addr = a.log_guest_addr;
1379                 vq->used = (void __user *)(unsigned long)a.used_user_addr;
1380                 break;
1381         case VHOST_SET_VRING_KICK:
1382                 if (copy_from_user(&f, argp, sizeof f)) {
1383                         r = -EFAULT;
1384                         break;
1385                 }
1386                 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1387                 if (IS_ERR(eventfp)) {
1388                         r = PTR_ERR(eventfp);
1389                         break;
1390                 }
1391                 if (eventfp != vq->kick) {
1392                         pollstop = (filep = vq->kick) != NULL;
1393                         pollstart = (vq->kick = eventfp) != NULL;
1394                 } else
1395                         filep = eventfp;
1396                 break;
1397         case VHOST_SET_VRING_CALL:
1398                 if (copy_from_user(&f, argp, sizeof f)) {
1399                         r = -EFAULT;
1400                         break;
1401                 }
1402                 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1403                 if (IS_ERR(eventfp)) {
1404                         r = PTR_ERR(eventfp);
1405                         break;
1406                 }
1407                 if (eventfp != vq->call) {
1408                         filep = vq->call;
1409                         ctx = vq->call_ctx;
1410                         vq->call = eventfp;
1411                         vq->call_ctx = eventfp ?
1412                                 eventfd_ctx_fileget(eventfp) : NULL;
1413                 } else
1414                         filep = eventfp;
1415                 break;
1416         case VHOST_SET_VRING_ERR:
1417                 if (copy_from_user(&f, argp, sizeof f)) {
1418                         r = -EFAULT;
1419                         break;
1420                 }
1421                 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1422                 if (IS_ERR(eventfp)) {
1423                         r = PTR_ERR(eventfp);
1424                         break;
1425                 }
1426                 if (eventfp != vq->error) {
1427                         filep = vq->error;
1428                         vq->error = eventfp;
1429                         ctx = vq->error_ctx;
1430                         vq->error_ctx = eventfp ?
1431                                 eventfd_ctx_fileget(eventfp) : NULL;
1432                 } else
1433                         filep = eventfp;
1434                 break;
1435         case VHOST_SET_VRING_ENDIAN:
1436                 r = vhost_set_vring_endian(vq, argp);
1437                 break;
1438         case VHOST_GET_VRING_ENDIAN:
1439                 r = vhost_get_vring_endian(vq, idx, argp);
1440                 break;
1441         case VHOST_SET_VRING_BUSYLOOP_TIMEOUT:
1442                 if (copy_from_user(&s, argp, sizeof(s))) {
1443                         r = -EFAULT;
1444                         break;
1445                 }
1446                 vq->busyloop_timeout = s.num;
1447                 break;
1448         case VHOST_GET_VRING_BUSYLOOP_TIMEOUT:
1449                 s.index = idx;
1450                 s.num = vq->busyloop_timeout;
1451                 if (copy_to_user(argp, &s, sizeof(s)))
1452                         r = -EFAULT;
1453                 break;
1454         default:
1455                 r = -ENOIOCTLCMD;
1456         }
1457
1458         if (pollstop && vq->handle_kick)
1459                 vhost_poll_stop(&vq->poll);
1460
1461         if (ctx)
1462                 eventfd_ctx_put(ctx);
1463         if (filep)
1464                 fput(filep);
1465
1466         if (pollstart && vq->handle_kick)
1467                 r = vhost_poll_start(&vq->poll, vq->kick);
1468
1469         mutex_unlock(&vq->mutex);
1470
1471         if (pollstop && vq->handle_kick)
1472                 vhost_poll_flush(&vq->poll);
1473         return r;
1474 }
1475 EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
1476
1477 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
1478 {
1479         struct vhost_umem *niotlb, *oiotlb;
1480         int i;
1481
1482         niotlb = vhost_umem_alloc();
1483         if (!niotlb)
1484                 return -ENOMEM;
1485
1486         oiotlb = d->iotlb;
1487         d->iotlb = niotlb;
1488
1489         for (i = 0; i < d->nvqs; ++i) {
1490                 mutex_lock(&d->vqs[i]->mutex);
1491                 d->vqs[i]->iotlb = niotlb;
1492                 mutex_unlock(&d->vqs[i]->mutex);
1493         }
1494
1495         vhost_umem_clean(oiotlb);
1496
1497         return 0;
1498 }
1499 EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
1500
1501 /* Caller must have device mutex */
1502 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
1503 {
1504         struct file *eventfp, *filep = NULL;
1505         struct eventfd_ctx *ctx = NULL;
1506         u64 p;
1507         long r;
1508         int i, fd;
1509
1510         /* If you are not the owner, you can become one */
1511         if (ioctl == VHOST_SET_OWNER) {
1512                 r = vhost_dev_set_owner(d);
1513                 goto done;
1514         }
1515
1516         /* You must be the owner to do anything else */
1517         r = vhost_dev_check_owner(d);
1518         if (r)
1519                 goto done;
1520
1521         switch (ioctl) {
1522         case VHOST_SET_MEM_TABLE:
1523                 r = vhost_set_memory(d, argp);
1524                 break;
1525         case VHOST_SET_LOG_BASE:
1526                 if (copy_from_user(&p, argp, sizeof p)) {
1527                         r = -EFAULT;
1528                         break;
1529                 }
1530                 if ((u64)(unsigned long)p != p) {
1531                         r = -EFAULT;
1532                         break;
1533                 }
1534                 for (i = 0; i < d->nvqs; ++i) {
1535                         struct vhost_virtqueue *vq;
1536                         void __user *base = (void __user *)(unsigned long)p;
1537                         vq = d->vqs[i];
1538                         mutex_lock(&vq->mutex);
1539                         /* If ring is inactive, will check when it's enabled. */
1540                         if (vq->private_data && !vq_log_access_ok(vq, base))
1541                                 r = -EFAULT;
1542                         else
1543                                 vq->log_base = base;
1544                         mutex_unlock(&vq->mutex);
1545                 }
1546                 break;
1547         case VHOST_SET_LOG_FD:
1548                 r = get_user(fd, (int __user *)argp);
1549                 if (r < 0)
1550                         break;
1551                 eventfp = fd == -1 ? NULL : eventfd_fget(fd);
1552                 if (IS_ERR(eventfp)) {
1553                         r = PTR_ERR(eventfp);
1554                         break;
1555                 }
1556                 if (eventfp != d->log_file) {
1557                         filep = d->log_file;
1558                         d->log_file = eventfp;
1559                         ctx = d->log_ctx;
1560                         d->log_ctx = eventfp ?
1561                                 eventfd_ctx_fileget(eventfp) : NULL;
1562                 } else
1563                         filep = eventfp;
1564                 for (i = 0; i < d->nvqs; ++i) {
1565                         mutex_lock(&d->vqs[i]->mutex);
1566                         d->vqs[i]->log_ctx = d->log_ctx;
1567                         mutex_unlock(&d->vqs[i]->mutex);
1568                 }
1569                 if (ctx)
1570                         eventfd_ctx_put(ctx);
1571                 if (filep)
1572                         fput(filep);
1573                 break;
1574         default:
1575                 r = -ENOIOCTLCMD;
1576                 break;
1577         }
1578 done:
1579         return r;
1580 }
1581 EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
1582
1583 /* TODO: This is really inefficient.  We need something like get_user()
1584  * (instruction directly accesses the data, with an exception table entry
1585  * returning -EFAULT). See Documentation/x86/exception-tables.txt.
1586  */
1587 static int set_bit_to_user(int nr, void __user *addr)
1588 {
1589         unsigned long log = (unsigned long)addr;
1590         struct page *page;
1591         void *base;
1592         int bit = nr + (log % PAGE_SIZE) * 8;
1593         int r;
1594
1595         r = get_user_pages_fast(log, 1, 1, &page);
1596         if (r < 0)
1597                 return r;
1598         BUG_ON(r != 1);
1599         base = kmap_atomic(page);
1600         set_bit(bit, base);
1601         kunmap_atomic(base);
1602         set_page_dirty_lock(page);
1603         put_page(page);
1604         return 0;
1605 }
1606
1607 static int log_write(void __user *log_base,
1608                      u64 write_address, u64 write_length)
1609 {
1610         u64 write_page = write_address / VHOST_PAGE_SIZE;
1611         int r;
1612
1613         if (!write_length)
1614                 return 0;
1615         write_length += write_address % VHOST_PAGE_SIZE;
1616         for (;;) {
1617                 u64 base = (u64)(unsigned long)log_base;
1618                 u64 log = base + write_page / 8;
1619                 int bit = write_page % 8;
1620                 if ((u64)(unsigned long)log != log)
1621                         return -EFAULT;
1622                 r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
1623                 if (r < 0)
1624                         return r;
1625                 if (write_length <= VHOST_PAGE_SIZE)
1626                         break;
1627                 write_length -= VHOST_PAGE_SIZE;
1628                 write_page += 1;
1629         }
1630         return r;
1631 }
1632
1633 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
1634                     unsigned int log_num, u64 len)
1635 {
1636         int i, r;
1637
1638         /* Make sure data written is seen before log. */
1639         smp_wmb();
1640         for (i = 0; i < log_num; ++i) {
1641                 u64 l = min(log[i].len, len);
1642                 r = log_write(vq->log_base, log[i].addr, l);
1643                 if (r < 0)
1644                         return r;
1645                 len -= l;
1646                 if (!len) {
1647                         if (vq->log_ctx)
1648                                 eventfd_signal(vq->log_ctx, 1);
1649                         return 0;
1650                 }
1651         }
1652         /* Length written exceeds what we have stored. This is a bug. */
1653         BUG();
1654         return 0;
1655 }
1656 EXPORT_SYMBOL_GPL(vhost_log_write);
1657
1658 static int vhost_update_used_flags(struct vhost_virtqueue *vq)
1659 {
1660         void __user *used;
1661         if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1662                            &vq->used->flags) < 0)
1663                 return -EFAULT;
1664         if (unlikely(vq->log_used)) {
1665                 /* Make sure the flag is seen before log. */
1666                 smp_wmb();
1667                 /* Log used flag write. */
1668                 used = &vq->used->flags;
1669                 log_write(vq->log_base, vq->log_addr +
1670                           (used - (void __user *)vq->used),
1671                           sizeof vq->used->flags);
1672                 if (vq->log_ctx)
1673                         eventfd_signal(vq->log_ctx, 1);
1674         }
1675         return 0;
1676 }
1677
1678 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
1679 {
1680         if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1681                            vhost_avail_event(vq)))
1682                 return -EFAULT;
1683         if (unlikely(vq->log_used)) {
1684                 void __user *used;
1685                 /* Make sure the event is seen before log. */
1686                 smp_wmb();
1687                 /* Log avail event write */
1688                 used = vhost_avail_event(vq);
1689                 log_write(vq->log_base, vq->log_addr +
1690                           (used - (void __user *)vq->used),
1691                           sizeof *vhost_avail_event(vq));
1692                 if (vq->log_ctx)
1693                         eventfd_signal(vq->log_ctx, 1);
1694         }
1695         return 0;
1696 }
1697
1698 int vhost_vq_init_access(struct vhost_virtqueue *vq)
1699 {
1700         __virtio16 last_used_idx;
1701         int r;
1702         bool is_le = vq->is_le;
1703
1704         if (!vq->private_data) {
1705                 vhost_reset_is_le(vq);
1706                 return 0;
1707         }
1708
1709         vhost_init_is_le(vq);
1710
1711         r = vhost_update_used_flags(vq);
1712         if (r)
1713                 goto err;
1714         vq->signalled_used_valid = false;
1715         if (!vq->iotlb &&
1716             !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) {
1717                 r = -EFAULT;
1718                 goto err;
1719         }
1720         r = vhost_get_user(vq, last_used_idx, &vq->used->idx);
1721         if (r) {
1722                 vq_err(vq, "Can't access used idx at %p\n",
1723                        &vq->used->idx);
1724                 goto err;
1725         }
1726         vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
1727         return 0;
1728
1729 err:
1730         vq->is_le = is_le;
1731         return r;
1732 }
1733 EXPORT_SYMBOL_GPL(vhost_vq_init_access);
1734
1735 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1736                           struct iovec iov[], int iov_size, int access)
1737 {
1738         const struct vhost_umem_node *node;
1739         struct vhost_dev *dev = vq->dev;
1740         struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;
1741         struct iovec *_iov;
1742         u64 s = 0;
1743         int ret = 0;
1744
1745         while ((u64)len > s) {
1746                 u64 size;
1747                 if (unlikely(ret >= iov_size)) {
1748                         ret = -ENOBUFS;
1749                         break;
1750                 }
1751
1752                 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1753                                                         addr, addr + len - 1);
1754                 if (node == NULL || node->start > addr) {
1755                         if (umem != dev->iotlb) {
1756                                 ret = -EFAULT;
1757                                 break;
1758                         }
1759                         ret = -EAGAIN;
1760                         break;
1761                 } else if (!(node->perm & access)) {
1762                         ret = -EPERM;
1763                         break;
1764                 }
1765
1766                 _iov = iov + ret;
1767                 size = node->size - addr + node->start;
1768                 _iov->iov_len = min((u64)len - s, size);
1769                 _iov->iov_base = (void __user *)(unsigned long)
1770                         (node->userspace_addr + addr - node->start);
1771                 s += size;
1772                 addr += size;
1773                 ++ret;
1774         }
1775
1776         if (ret == -EAGAIN)
1777                 vhost_iotlb_miss(vq, addr, access);
1778         return ret;
1779 }
1780
1781 /* Each buffer in the virtqueues is actually a chain of descriptors.  This
1782  * function returns the next descriptor in the chain,
1783  * or -1U if we're at the end. */
1784 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
1785 {
1786         unsigned int next;
1787
1788         /* If this descriptor says it doesn't chain, we're done. */
1789         if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
1790                 return -1U;
1791
1792         /* Check they're not leading us off end of descriptors. */
1793         next = vhost16_to_cpu(vq, desc->next);
1794         /* Make sure compiler knows to grab that: we don't want it changing! */
1795         /* We will use the result as an index in an array, so most
1796          * architectures only need a compiler barrier here. */
1797         read_barrier_depends();
1798
1799         return next;
1800 }
1801
1802 static int get_indirect(struct vhost_virtqueue *vq,
1803                         struct iovec iov[], unsigned int iov_size,
1804                         unsigned int *out_num, unsigned int *in_num,
1805                         struct vhost_log *log, unsigned int *log_num,
1806                         struct vring_desc *indirect)
1807 {
1808         struct vring_desc desc;
1809         unsigned int i = 0, count, found = 0;
1810         u32 len = vhost32_to_cpu(vq, indirect->len);
1811         struct iov_iter from;
1812         int ret, access;
1813
1814         /* Sanity check */
1815         if (unlikely(len % sizeof desc)) {
1816                 vq_err(vq, "Invalid length in indirect descriptor: "
1817                        "len 0x%llx not multiple of 0x%zx\n",
1818                        (unsigned long long)len,
1819                        sizeof desc);
1820                 return -EINVAL;
1821         }
1822
1823         ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
1824                              UIO_MAXIOV, VHOST_ACCESS_RO);
1825         if (unlikely(ret < 0)) {
1826                 if (ret != -EAGAIN)
1827                         vq_err(vq, "Translation failure %d in indirect.\n", ret);
1828                 return ret;
1829         }
1830         iov_iter_init(&from, READ, vq->indirect, ret, len);
1831
1832         /* We will use the result as an address to read from, so most
1833          * architectures only need a compiler barrier here. */
1834         read_barrier_depends();
1835
1836         count = len / sizeof desc;
1837         /* Buffers are chained via a 16 bit next field, so
1838          * we can have at most 2^16 of these. */
1839         if (unlikely(count > USHRT_MAX + 1)) {
1840                 vq_err(vq, "Indirect buffer length too big: %d\n",
1841                        indirect->len);
1842                 return -E2BIG;
1843         }
1844
1845         do {
1846                 unsigned iov_count = *in_num + *out_num;
1847                 if (unlikely(++found > count)) {
1848                         vq_err(vq, "Loop detected: last one at %u "
1849                                "indirect size %u\n",
1850                                i, count);
1851                         return -EINVAL;
1852                 }
1853                 if (unlikely(copy_from_iter(&desc, sizeof(desc), &from) !=
1854                              sizeof(desc))) {
1855                         vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
1856                                i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1857                         return -EINVAL;
1858                 }
1859                 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
1860                         vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
1861                                i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1862                         return -EINVAL;
1863                 }
1864
1865                 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
1866                         access = VHOST_ACCESS_WO;
1867                 else
1868                         access = VHOST_ACCESS_RO;
1869
1870                 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
1871                                      vhost32_to_cpu(vq, desc.len), iov + iov_count,
1872                                      iov_size - iov_count, access);
1873                 if (unlikely(ret < 0)) {
1874                         if (ret != -EAGAIN)
1875                                 vq_err(vq, "Translation failure %d indirect idx %d\n",
1876                                         ret, i);
1877                         return ret;
1878                 }
1879                 /* If this is an input descriptor, increment that count. */
1880                 if (access == VHOST_ACCESS_WO) {
1881                         *in_num += ret;
1882                         if (unlikely(log)) {
1883                                 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
1884                                 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1885                                 ++*log_num;
1886                         }
1887                 } else {
1888                         /* If it's an output descriptor, they're all supposed
1889                          * to come before any input descriptors. */
1890                         if (unlikely(*in_num)) {
1891                                 vq_err(vq, "Indirect descriptor "
1892                                        "has out after in: idx %d\n", i);
1893                                 return -EINVAL;
1894                         }
1895                         *out_num += ret;
1896                 }
1897         } while ((i = next_desc(vq, &desc)) != -1);
1898         return 0;
1899 }
1900
1901 /* This looks in the virtqueue and for the first available buffer, and converts
1902  * it to an iovec for convenient access.  Since descriptors consist of some
1903  * number of output then some number of input descriptors, it's actually two
1904  * iovecs, but we pack them into one and note how many of each there were.
1905  *
1906  * This function returns the descriptor number found, or vq->num (which is
1907  * never a valid descriptor number) if none was found.  A negative code is
1908  * returned on error. */
1909 int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1910                       struct iovec iov[], unsigned int iov_size,
1911                       unsigned int *out_num, unsigned int *in_num,
1912                       struct vhost_log *log, unsigned int *log_num)
1913 {
1914         struct vring_desc desc;
1915         unsigned int i, head, found = 0;
1916         u16 last_avail_idx;
1917         __virtio16 avail_idx;
1918         __virtio16 ring_head;
1919         int ret, access;
1920
1921         /* Check it isn't doing very strange things with descriptor numbers. */
1922         last_avail_idx = vq->last_avail_idx;
1923         if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) {
1924                 vq_err(vq, "Failed to access avail idx at %p\n",
1925                        &vq->avail->idx);
1926                 return -EFAULT;
1927         }
1928         vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
1929
1930         if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
1931                 vq_err(vq, "Guest moved used index from %u to %u",
1932                        last_avail_idx, vq->avail_idx);
1933                 return -EFAULT;
1934         }
1935
1936         /* If there's nothing new since last we looked, return invalid. */
1937         if (vq->avail_idx == last_avail_idx)
1938                 return vq->num;
1939
1940         /* Only get avail ring entries after they have been exposed by guest. */
1941         smp_rmb();
1942
1943         /* Grab the next descriptor number they're advertising, and increment
1944          * the index we've seen. */
1945         if (unlikely(vhost_get_user(vq, ring_head,
1946                      &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
1947                 vq_err(vq, "Failed to read head: idx %d address %p\n",
1948                        last_avail_idx,
1949                        &vq->avail->ring[last_avail_idx % vq->num]);
1950                 return -EFAULT;
1951         }
1952
1953         head = vhost16_to_cpu(vq, ring_head);
1954
1955         /* If their number is silly, that's an error. */
1956         if (unlikely(head >= vq->num)) {
1957                 vq_err(vq, "Guest says index %u > %u is available",
1958                        head, vq->num);
1959                 return -EINVAL;
1960         }
1961
1962         /* When we start there are none of either input nor output. */
1963         *out_num = *in_num = 0;
1964         if (unlikely(log))
1965                 *log_num = 0;
1966
1967         i = head;
1968         do {
1969                 unsigned iov_count = *in_num + *out_num;
1970                 if (unlikely(i >= vq->num)) {
1971                         vq_err(vq, "Desc index is %u > %u, head = %u",
1972                                i, vq->num, head);
1973                         return -EINVAL;
1974                 }
1975                 if (unlikely(++found > vq->num)) {
1976                         vq_err(vq, "Loop detected: last one at %u "
1977                                "vq size %u head %u\n",
1978                                i, vq->num, head);
1979                         return -EINVAL;
1980                 }
1981                 ret = vhost_copy_from_user(vq, &desc, vq->desc + i,
1982                                            sizeof desc);
1983                 if (unlikely(ret)) {
1984                         vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
1985                                i, vq->desc + i);
1986                         return -EFAULT;
1987                 }
1988                 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
1989                         ret = get_indirect(vq, iov, iov_size,
1990                                            out_num, in_num,
1991                                            log, log_num, &desc);
1992                         if (unlikely(ret < 0)) {
1993                                 if (ret != -EAGAIN)
1994                                         vq_err(vq, "Failure detected "
1995                                                 "in indirect descriptor at idx %d\n", i);
1996                                 return ret;
1997                         }
1998                         continue;
1999                 }
2000
2001                 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2002                         access = VHOST_ACCESS_WO;
2003                 else
2004                         access = VHOST_ACCESS_RO;
2005                 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2006                                      vhost32_to_cpu(vq, desc.len), iov + iov_count,
2007                                      iov_size - iov_count, access);
2008                 if (unlikely(ret < 0)) {
2009                         if (ret != -EAGAIN)
2010                                 vq_err(vq, "Translation failure %d descriptor idx %d\n",
2011                                         ret, i);
2012                         return ret;
2013                 }
2014                 if (access == VHOST_ACCESS_WO) {
2015                         /* If this is an input descriptor,
2016                          * increment that count. */
2017                         *in_num += ret;
2018                         if (unlikely(log)) {
2019                                 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2020                                 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
2021                                 ++*log_num;
2022                         }
2023                 } else {
2024                         /* If it's an output descriptor, they're all supposed
2025                          * to come before any input descriptors. */
2026                         if (unlikely(*in_num)) {
2027                                 vq_err(vq, "Descriptor has out after in: "
2028                                        "idx %d\n", i);
2029                                 return -EINVAL;
2030                         }
2031                         *out_num += ret;
2032                 }
2033         } while ((i = next_desc(vq, &desc)) != -1);
2034
2035         /* On success, increment avail index. */
2036         vq->last_avail_idx++;
2037
2038         /* Assume notifications from guest are disabled at this point,
2039          * if they aren't we would need to update avail_event index. */
2040         BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
2041         return head;
2042 }
2043 EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
2044
2045 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
2046 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
2047 {
2048         vq->last_avail_idx -= n;
2049 }
2050 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
2051
2052 /* After we've used one of their buffers, we tell them about it.  We'll then
2053  * want to notify the guest, using eventfd. */
2054 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
2055 {
2056         struct vring_used_elem heads = {
2057                 cpu_to_vhost32(vq, head),
2058                 cpu_to_vhost32(vq, len)
2059         };
2060
2061         return vhost_add_used_n(vq, &heads, 1);
2062 }
2063 EXPORT_SYMBOL_GPL(vhost_add_used);
2064
2065 static int __vhost_add_used_n(struct vhost_virtqueue *vq,
2066                             struct vring_used_elem *heads,
2067                             unsigned count)
2068 {
2069         struct vring_used_elem __user *used;
2070         u16 old, new;
2071         int start;
2072
2073         start = vq->last_used_idx & (vq->num - 1);
2074         used = vq->used->ring + start;
2075         if (count == 1) {
2076                 if (vhost_put_user(vq, heads[0].id, &used->id)) {
2077                         vq_err(vq, "Failed to write used id");
2078                         return -EFAULT;
2079                 }
2080                 if (vhost_put_user(vq, heads[0].len, &used->len)) {
2081                         vq_err(vq, "Failed to write used len");
2082                         return -EFAULT;
2083                 }
2084         } else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) {
2085                 vq_err(vq, "Failed to write used");
2086                 return -EFAULT;
2087         }
2088         if (unlikely(vq->log_used)) {
2089                 /* Make sure data is seen before log. */
2090                 smp_wmb();
2091                 /* Log used ring entry write. */
2092                 log_write(vq->log_base,
2093                           vq->log_addr +
2094                            ((void __user *)used - (void __user *)vq->used),
2095                           count * sizeof *used);
2096         }
2097         old = vq->last_used_idx;
2098         new = (vq->last_used_idx += count);
2099         /* If the driver never bothers to signal in a very long while,
2100          * used index might wrap around. If that happens, invalidate
2101          * signalled_used index we stored. TODO: make sure driver
2102          * signals at least once in 2^16 and remove this. */
2103         if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
2104                 vq->signalled_used_valid = false;
2105         return 0;
2106 }
2107
2108 /* After we've used one of their buffers, we tell them about it.  We'll then
2109  * want to notify the guest, using eventfd. */
2110 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
2111                      unsigned count)
2112 {
2113         int start, n, r;
2114
2115         start = vq->last_used_idx & (vq->num - 1);
2116         n = vq->num - start;
2117         if (n < count) {
2118                 r = __vhost_add_used_n(vq, heads, n);
2119                 if (r < 0)
2120                         return r;
2121                 heads += n;
2122                 count -= n;
2123         }
2124         r = __vhost_add_used_n(vq, heads, count);
2125
2126         /* Make sure buffer is written before we update index. */
2127         smp_wmb();
2128         if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
2129                            &vq->used->idx)) {
2130                 vq_err(vq, "Failed to increment used idx");
2131                 return -EFAULT;
2132         }
2133         if (unlikely(vq->log_used)) {
2134                 /* Log used index update. */
2135                 log_write(vq->log_base,
2136                           vq->log_addr + offsetof(struct vring_used, idx),
2137                           sizeof vq->used->idx);
2138                 if (vq->log_ctx)
2139                         eventfd_signal(vq->log_ctx, 1);
2140         }
2141         return r;
2142 }
2143 EXPORT_SYMBOL_GPL(vhost_add_used_n);
2144
2145 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2146 {
2147         __u16 old, new;
2148         __virtio16 event;
2149         bool v;
2150         /* Flush out used index updates. This is paired
2151          * with the barrier that the Guest executes when enabling
2152          * interrupts. */
2153         smp_mb();
2154
2155         if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2156             unlikely(vq->avail_idx == vq->last_avail_idx))
2157                 return true;
2158
2159         if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2160                 __virtio16 flags;
2161                 if (vhost_get_user(vq, flags, &vq->avail->flags)) {
2162                         vq_err(vq, "Failed to get flags");
2163                         return true;
2164                 }
2165                 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
2166         }
2167         old = vq->signalled_used;
2168         v = vq->signalled_used_valid;
2169         new = vq->signalled_used = vq->last_used_idx;
2170         vq->signalled_used_valid = true;
2171
2172         if (unlikely(!v))
2173                 return true;
2174
2175         if (vhost_get_user(vq, event, vhost_used_event(vq))) {
2176                 vq_err(vq, "Failed to get used event idx");
2177                 return true;
2178         }
2179         return vring_need_event(vhost16_to_cpu(vq, event), new, old);
2180 }
2181
2182 /* This actually signals the guest, using eventfd. */
2183 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2184 {
2185         /* Signal the Guest tell them we used something up. */
2186         if (vq->call_ctx && vhost_notify(dev, vq))
2187                 eventfd_signal(vq->call_ctx, 1);
2188 }
2189 EXPORT_SYMBOL_GPL(vhost_signal);
2190
2191 /* And here's the combo meal deal.  Supersize me! */
2192 void vhost_add_used_and_signal(struct vhost_dev *dev,
2193                                struct vhost_virtqueue *vq,
2194                                unsigned int head, int len)
2195 {
2196         vhost_add_used(vq, head, len);
2197         vhost_signal(dev, vq);
2198 }
2199 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
2200
2201 /* multi-buffer version of vhost_add_used_and_signal */
2202 void vhost_add_used_and_signal_n(struct vhost_dev *dev,
2203                                  struct vhost_virtqueue *vq,
2204                                  struct vring_used_elem *heads, unsigned count)
2205 {
2206         vhost_add_used_n(vq, heads, count);
2207         vhost_signal(dev, vq);
2208 }
2209 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
2210
2211 /* return true if we're sure that avaiable ring is empty */
2212 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2213 {
2214         __virtio16 avail_idx;
2215         int r;
2216
2217         r = vhost_get_user(vq, avail_idx, &vq->avail->idx);
2218         if (r)
2219                 return false;
2220
2221         return vhost16_to_cpu(vq, avail_idx) == vq->avail_idx;
2222 }
2223 EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
2224
2225 /* OK, now we need to know about added descriptors. */
2226 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2227 {
2228         __virtio16 avail_idx;
2229         int r;
2230
2231         if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
2232                 return false;
2233         vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
2234         if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2235                 r = vhost_update_used_flags(vq);
2236                 if (r) {
2237                         vq_err(vq, "Failed to enable notification at %p: %d\n",
2238                                &vq->used->flags, r);
2239                         return false;
2240                 }
2241         } else {
2242                 r = vhost_update_avail_event(vq, vq->avail_idx);
2243                 if (r) {
2244                         vq_err(vq, "Failed to update avail event index at %p: %d\n",
2245                                vhost_avail_event(vq), r);
2246                         return false;
2247                 }
2248         }
2249         /* They could have slipped one in as we were doing that: make
2250          * sure it's written, then check again. */
2251         smp_mb();
2252         r = vhost_get_user(vq, avail_idx, &vq->avail->idx);
2253         if (r) {
2254                 vq_err(vq, "Failed to check avail idx at %p: %d\n",
2255                        &vq->avail->idx, r);
2256                 return false;
2257         }
2258
2259         return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
2260 }
2261 EXPORT_SYMBOL_GPL(vhost_enable_notify);
2262
2263 /* We don't need to be notified again. */
2264 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2265 {
2266         int r;
2267
2268         if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
2269                 return;
2270         vq->used_flags |= VRING_USED_F_NO_NOTIFY;
2271         if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2272                 r = vhost_update_used_flags(vq);
2273                 if (r)
2274                         vq_err(vq, "Failed to enable notification at %p: %d\n",
2275                                &vq->used->flags, r);
2276         }
2277 }
2278 EXPORT_SYMBOL_GPL(vhost_disable_notify);
2279
2280 /* Create a new message. */
2281 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type)
2282 {
2283         struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL);
2284         if (!node)
2285                 return NULL;
2286         node->vq = vq;
2287         node->msg.type = type;
2288         return node;
2289 }
2290 EXPORT_SYMBOL_GPL(vhost_new_msg);
2291
2292 void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head,
2293                        struct vhost_msg_node *node)
2294 {
2295         spin_lock(&dev->iotlb_lock);
2296         list_add_tail(&node->node, head);
2297         spin_unlock(&dev->iotlb_lock);
2298
2299         wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM);
2300 }
2301 EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
2302
2303 struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev,
2304                                          struct list_head *head)
2305 {
2306         struct vhost_msg_node *node = NULL;
2307
2308         spin_lock(&dev->iotlb_lock);
2309         if (!list_empty(head)) {
2310                 node = list_first_entry(head, struct vhost_msg_node,
2311                                         node);
2312                 list_del(&node->node);
2313         }
2314         spin_unlock(&dev->iotlb_lock);
2315
2316         return node;
2317 }
2318 EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
2319
2320
2321 static int __init vhost_init(void)
2322 {
2323         return 0;
2324 }
2325
2326 static void __exit vhost_exit(void)
2327 {
2328 }
2329
2330 module_init(vhost_init);
2331 module_exit(vhost_exit);
2332
2333 MODULE_VERSION("0.0.1");
2334 MODULE_LICENSE("GPL v2");
2335 MODULE_AUTHOR("Michael S. Tsirkin");
2336 MODULE_DESCRIPTION("Host kernel accelerator for virtio");