drivers/media/v4l2-core/videobuf2-core.c

   1 /*
   2  * videobuf2-core.c - video buffer 2 core framework
   3  *
   4  * Copyright (C) 2010 Samsung Electronics
   5  *
   6  * Author: Pawel Osciak <pawel@osciak.com>
   7  *         Marek Szyprowski <m.szyprowski@samsung.com>
   8  *
   9  * The vb2_thread implementation was based on code from videobuf-dvb.c:
  10  *      (c) 2004 Gerd Knorr <kraxel@bytesex.org> [SUSE Labs]
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation.
  15  */
  16
  17 #include <linux/err.h>
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/mm.h>
  21 #include <linux/poll.h>
  22 #include <linux/slab.h>
  23 #include <linux/sched.h>
  24 #include <linux/freezer.h>
  25 #include <linux/kthread.h>
  26
  27 #include <media/videobuf2-core.h>
  28 #include <media/v4l2-mc.h>
  29
  30 #include <trace/events/vb2.h>
  31
  32 static int debug;
  33 module_param(debug, int, 0644);
  34
  35 #define dprintk(level, fmt, arg...)                                           \
  36         do {                                                                  \
  37                 if (debug >= level)                                           \
  38                         pr_info("vb2-core: %s: " fmt, __func__, ## arg); \
  39         } while (0)
  40
  41 #ifdef CONFIG_VIDEO_ADV_DEBUG
  42
  43 /*
  44  * If advanced debugging is on, then count how often each op is called
  45  * successfully, which can either be per-buffer or per-queue.
  46  *
  47  * This makes it easy to check that the 'init' and 'cleanup'
  48  * (and variations thereof) stay balanced.
  49  */
  50
  51 #define log_memop(vb, op)                                               \
  52         dprintk(2, "call_memop(%p, %d, %s)%s\n",                        \
  53                 (vb)->vb2_queue, (vb)->index, #op,                      \
  54                 (vb)->vb2_queue->mem_ops->op ? "" : " (nop)")
  55
  56 #define call_memop(vb, op, args...)                                     \
  57 ({                                                                      \
  58         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  59         int err;                                                        \
  60                                                                         \
  61         log_memop(vb, op);                                              \
  62         err = _q->mem_ops->op ? _q->mem_ops->op(args) : 0;              \
  63         if (!err)                                                       \
  64                 (vb)->cnt_mem_ ## op++;                                 \
  65         err;                                                            \
  66 })
  67
  68 #define call_ptr_memop(vb, op, args...)                                 \
  69 ({                                                                      \
  70         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  71         void *ptr;                                                      \
  72                                                                         \
  73         log_memop(vb, op);                                              \
  74         ptr = _q->mem_ops->op ? _q->mem_ops->op(args) : NULL;           \
  75         if (!IS_ERR_OR_NULL(ptr))                                       \
  76                 (vb)->cnt_mem_ ## op++;                                 \
  77         ptr;                                                            \
  78 })
  79
  80 #define call_void_memop(vb, op, args...)                                \
  81 ({                                                                      \
  82         struct vb2_queue *_q = (vb)->vb2_queue;                         \
  83                                                                         \
  84         log_memop(vb, op);                                              \
  85         if (_q->mem_ops->op)                                            \
  86                 _q->mem_ops->op(args);                                  \
  87         (vb)->cnt_mem_ ## op++;                                         \
  88 })
  89
  90 #define log_qop(q, op)                                                  \
  91         dprintk(2, "call_qop(%p, %s)%s\n", q, #op,                      \
  92                 (q)->ops->op ? "" : " (nop)")
  93
  94 #define call_qop(q, op, args...)                                        \
  95 ({                                                                      \
  96         int err;                                                        \
  97                                                                         \
  98         log_qop(q, op);                                                 \
  99         err = (q)->ops->op ? (q)->ops->op(args) : 0;                    \
 100         if (!err)                                                       \
 101                 (q)->cnt_ ## op++;                                      \
 102         err;                                                            \
 103 })
 104
 105 #define call_void_qop(q, op, args...)                                   \
 106 ({                                                                      \
 107         log_qop(q, op);                                                 \
 108         if ((q)->ops->op)                                               \
 109                 (q)->ops->op(args);                                     \
 110         (q)->cnt_ ## op++;                                              \
 111 })
 112
 113 #define log_vb_qop(vb, op, args...)                                     \
 114         dprintk(2, "call_vb_qop(%p, %d, %s)%s\n",                       \
 115                 (vb)->vb2_queue, (vb)->index, #op,                      \
 116                 (vb)->vb2_queue->ops->op ? "" : " (nop)")
 117
 118 #define call_vb_qop(vb, op, args...)                                    \
 119 ({                                                                      \
 120         int err;                                                        \
 121                                                                         \
 122         log_vb_qop(vb, op);                                             \
 123         err = (vb)->vb2_queue->ops->op ?                                \
 124                 (vb)->vb2_queue->ops->op(args) : 0;                     \
 125         if (!err)                                                       \
 126                 (vb)->cnt_ ## op++;                                     \
 127         err;                                                            \
 128 })
 129
 130 #define call_void_vb_qop(vb, op, args...)                               \
 131 ({                                                                      \
 132         log_vb_qop(vb, op);                                             \
 133         if ((vb)->vb2_queue->ops->op)                                   \
 134                 (vb)->vb2_queue->ops->op(args);                         \
 135         (vb)->cnt_ ## op++;                                             \
 136 })
 137
 138 #else
 139
 140 #define call_memop(vb, op, args...)                                     \
 141         ((vb)->vb2_queue->mem_ops->op ?                                 \
 142                 (vb)->vb2_queue->mem_ops->op(args) : 0)
 143
 144 #define call_ptr_memop(vb, op, args...)                                 \
 145         ((vb)->vb2_queue->mem_ops->op ?                                 \
 146                 (vb)->vb2_queue->mem_ops->op(args) : NULL)
 147
 148 #define call_void_memop(vb, op, args...)                                \
 149         do {                                                            \
 150                 if ((vb)->vb2_queue->mem_ops->op)                       \
 151                         (vb)->vb2_queue->mem_ops->op(args);             \
 152         } while (0)
 153
 154 #define call_qop(q, op, args...)                                        \
 155         ((q)->ops->op ? (q)->ops->op(args) : 0)
 156
 157 #define call_void_qop(q, op, args...)                                   \
 158         do {                                                            \
 159                 if ((q)->ops->op)                                       \
 160                         (q)->ops->op(args);                             \
 161         } while (0)
 162
 163 #define call_vb_qop(vb, op, args...)                                    \
 164         ((vb)->vb2_queue->ops->op ? (vb)->vb2_queue->ops->op(args) : 0)
 165
 166 #define call_void_vb_qop(vb, op, args...)                               \
 167         do {                                                            \
 168                 if ((vb)->vb2_queue->ops->op)                           \
 169                         (vb)->vb2_queue->ops->op(args);                 \
 170         } while (0)
 171
 172 #endif
 173
 174 #define call_bufop(q, op, args...)                                      \
 175 ({                                                                      \
 176         int ret = 0;                                                    \
 177         if (q && q->buf_ops && q->buf_ops->op)                          \
 178                 ret = q->buf_ops->op(args);                             \
 179         ret;                                                            \
 180 })
 181
 182 #define call_void_bufop(q, op, args...)                                 \
 183 ({                                                                      \
 184         if (q && q->buf_ops && q->buf_ops->op)                          \
 185                 q->buf_ops->op(args);                                   \
 186 })
 187
 188 static void __vb2_queue_cancel(struct vb2_queue *q);
 189 static void __enqueue_in_driver(struct vb2_buffer *vb);
 190
 191 /**
 192  * __vb2_buf_mem_alloc() - allocate video memory for the given buffer
 193  */
 194 static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 195 {
 196         struct vb2_queue *q = vb->vb2_queue;
 197         enum dma_data_direction dma_dir =
 198                 q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 199         void *mem_priv;
 200         int plane;
 201
 202         /*
 203          * Allocate memory for all planes in this buffer
 204          * NOTE: mmapped areas should be page aligned
 205          */
 206         for (plane = 0; plane < vb->num_planes; ++plane) {
 207                 unsigned long size = PAGE_ALIGN(vb->planes[plane].length);
 208
 209                 mem_priv = call_ptr_memop(vb, alloc,
 210                                 q->alloc_devs[plane] ? : q->dev,
 211                                 q->dma_attrs, size, dma_dir, q->gfp_flags);
 212                 if (IS_ERR_OR_NULL(mem_priv))
 213                         goto free;
 214
 215                 /* Associate allocator private data with this plane */
 216                 vb->planes[plane].mem_priv = mem_priv;
 217         }
 218
 219         return 0;
 220 free:
 221         /* Free already allocated memory if one of the allocations failed */
 222         for (; plane > 0; --plane) {
 223                 call_void_memop(vb, put, vb->planes[plane - 1].mem_priv);
 224                 vb->planes[plane - 1].mem_priv = NULL;
 225         }
 226
 227         return -ENOMEM;
 228 }
 229
 230 /**
 231  * __vb2_buf_mem_free() - free memory of the given buffer
 232  */
 233 static void __vb2_buf_mem_free(struct vb2_buffer *vb)
 234 {
 235         unsigned int plane;
 236
 237         for (plane = 0; plane < vb->num_planes; ++plane) {
 238                 call_void_memop(vb, put, vb->planes[plane].mem_priv);
 239                 vb->planes[plane].mem_priv = NULL;
 240                 dprintk(3, "freed plane %d of buffer %d\n", plane, vb->index);
 241         }
 242 }
 243
 244 /**
 245  * __vb2_buf_userptr_put() - release userspace memory associated with
 246  * a USERPTR buffer
 247  */
 248 static void __vb2_buf_userptr_put(struct vb2_buffer *vb)
 249 {
 250         unsigned int plane;
 251
 252         for (plane = 0; plane < vb->num_planes; ++plane) {
 253                 if (vb->planes[plane].mem_priv)
 254                         call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
 255                 vb->planes[plane].mem_priv = NULL;
 256         }
 257 }
 258
 259 /**
 260  * __vb2_plane_dmabuf_put() - release memory associated with
 261  * a DMABUF shared plane
 262  */
 263 static void __vb2_plane_dmabuf_put(struct vb2_buffer *vb, struct vb2_plane *p)
 264 {
 265         if (!p->mem_priv)
 266                 return;
 267
 268         if (p->dbuf_mapped)
 269                 call_void_memop(vb, unmap_dmabuf, p->mem_priv);
 270
 271         call_void_memop(vb, detach_dmabuf, p->mem_priv);
 272         dma_buf_put(p->dbuf);
 273         p->mem_priv = NULL;
 274         p->dbuf = NULL;
 275         p->dbuf_mapped = 0;
 276 }
 277
 278 /**
 279  * __vb2_buf_dmabuf_put() - release memory associated with
 280  * a DMABUF shared buffer
 281  */
 282 static void __vb2_buf_dmabuf_put(struct vb2_buffer *vb)
 283 {
 284         unsigned int plane;
 285
 286         for (plane = 0; plane < vb->num_planes; ++plane)
 287                 __vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
 288 }
 289
 290 /**
 291  * __setup_offsets() - setup unique offsets ("cookies") for every plane in
 292  * the buffer.
 293  */
 294 static void __setup_offsets(struct vb2_buffer *vb)
 295 {
 296         struct vb2_queue *q = vb->vb2_queue;
 297         unsigned int plane;
 298         unsigned long off = 0;
 299
 300         if (vb->index) {
 301                 struct vb2_buffer *prev = q->bufs[vb->index - 1];
 302                 struct vb2_plane *p = &prev->planes[prev->num_planes - 1];
 303
 304                 off = PAGE_ALIGN(p->m.offset + p->length);
 305         }
 306
 307         for (plane = 0; plane < vb->num_planes; ++plane) {
 308                 vb->planes[plane].m.offset = off;
 309
 310                 dprintk(3, "buffer %d, plane %d offset 0x%08lx\n",
 311                                 vb->index, plane, off);
 312
 313                 off += vb->planes[plane].length;
 314                 off = PAGE_ALIGN(off);
 315         }
 316 }
 317
 318 /**
 319  * __vb2_queue_alloc() - allocate videobuf buffer structures and (for MMAP type)
 320  * video buffer memory for all buffers/planes on the queue and initializes the
 321  * queue
 322  *
 323  * Returns the number of buffers successfully allocated.
 324  */
 325 static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory,
 326                              unsigned int num_buffers, unsigned int num_planes,
 327                              const unsigned plane_sizes[VB2_MAX_PLANES])
 328 {
 329         unsigned int buffer, plane;
 330         struct vb2_buffer *vb;
 331         int ret;
 332
 333         for (buffer = 0; buffer < num_buffers; ++buffer) {
 334                 /* Allocate videobuf buffer structures */
 335                 vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
 336                 if (!vb) {
 337                         dprintk(1, "memory alloc for buffer struct failed\n");
 338                         break;
 339                 }
 340
 341                 vb->state = VB2_BUF_STATE_DEQUEUED;
 342                 vb->vb2_queue = q;
 343                 vb->num_planes = num_planes;
 344                 vb->index = q->num_buffers + buffer;
 345                 vb->type = q->type;
 346                 vb->memory = memory;
 347                 for (plane = 0; plane < num_planes; ++plane) {
 348                         vb->planes[plane].length = plane_sizes[plane];
 349                         vb->planes[plane].min_length = plane_sizes[plane];
 350                 }
 351                 q->bufs[vb->index] = vb;
 352
 353                 /* Allocate video buffer memory for the MMAP type */
 354                 if (memory == VB2_MEMORY_MMAP) {
 355                         ret = __vb2_buf_mem_alloc(vb);
 356                         if (ret) {
 357                                 dprintk(1, "failed allocating memory for "
 358                                                 "buffer %d\n", buffer);
 359                                 q->bufs[vb->index] = NULL;
 360                                 kfree(vb);
 361                                 break;
 362                         }
 363                         __setup_offsets(vb);
 364                         /*
 365                          * Call the driver-provided buffer initialization
 366                          * callback, if given. An error in initialization
 367                          * results in queue setup failure.
 368                          */
 369                         ret = call_vb_qop(vb, buf_init, vb);
 370                         if (ret) {
 371                                 dprintk(1, "buffer %d %p initialization"
 372                                         " failed\n", buffer, vb);
 373                                 __vb2_buf_mem_free(vb);
 374                                 q->bufs[vb->index] = NULL;
 375                                 kfree(vb);
 376                                 break;
 377                         }
 378                 }
 379         }
 380
 381         dprintk(1, "allocated %d buffers, %d plane(s) each\n",
 382                         buffer, num_planes);
 383
 384         return buffer;
 385 }
 386
 387 /**
 388  * __vb2_free_mem() - release all video buffer memory for a given queue
 389  */
 390 static void __vb2_free_mem(struct vb2_queue *q, unsigned int buffers)
 391 {
 392         unsigned int buffer;
 393         struct vb2_buffer *vb;
 394
 395         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 396              ++buffer) {
 397                 vb = q->bufs[buffer];
 398                 if (!vb)
 399                         continue;
 400
 401                 /* Free MMAP buffers or release USERPTR buffers */
 402                 if (q->memory == VB2_MEMORY_MMAP)
 403                         __vb2_buf_mem_free(vb);
 404                 else if (q->memory == VB2_MEMORY_DMABUF)
 405                         __vb2_buf_dmabuf_put(vb);
 406                 else
 407                         __vb2_buf_userptr_put(vb);
 408         }
 409 }
 410
 411 /**
 412  * __vb2_queue_free() - free buffers at the end of the queue - video memory and
 413  * related information, if no buffers are left return the queue to an
 414  * uninitialized state. Might be called even if the queue has already been freed.
 415  */
 416 static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 417 {
 418         unsigned int buffer;
 419
 420         /*
 421          * Sanity check: when preparing a buffer the queue lock is released for
 422          * a short while (see __buf_prepare for the details), which would allow
 423          * a race with a reqbufs which can call this function. Removing the
 424          * buffers from underneath __buf_prepare is obviously a bad idea, so we
 425          * check if any of the buffers is in the state PREPARING, and if so we
 426          * just return -EAGAIN.
 427          */
 428         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 429              ++buffer) {
 430                 if (q->bufs[buffer] == NULL)
 431                         continue;
 432                 if (q->bufs[buffer]->state == VB2_BUF_STATE_PREPARING) {
 433                         dprintk(1, "preparing buffers, cannot free\n");
 434                         return -EAGAIN;
 435                 }
 436         }
 437
 438         /* Call driver-provided cleanup function for each buffer, if provided */
 439         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 440              ++buffer) {
 441                 struct vb2_buffer *vb = q->bufs[buffer];
 442
 443                 if (vb && vb->planes[0].mem_priv)
 444                         call_void_vb_qop(vb, buf_cleanup, vb);
 445         }
 446
 447         /* Release video buffer memory */
 448         __vb2_free_mem(q, buffers);
 449
 450 #ifdef CONFIG_VIDEO_ADV_DEBUG
 451         /*
 452          * Check that all the calls were balances during the life-time of this
 453          * queue. If not (or if the debug level is 1 or up), then dump the
 454          * counters to the kernel log.
 455          */
 456         if (q->num_buffers) {
 457                 bool unbalanced = q->cnt_start_streaming != q->cnt_stop_streaming ||
 458                                   q->cnt_wait_prepare != q->cnt_wait_finish;
 459
 460                 if (unbalanced || debug) {
 461                         pr_info("vb2: counters for queue %p:%s\n", q,
 462                                 unbalanced ? " UNBALANCED!" : "");
 463                         pr_info("vb2:     setup: %u start_streaming: %u stop_streaming: %u\n",
 464                                 q->cnt_queue_setup, q->cnt_start_streaming,
 465                                 q->cnt_stop_streaming);
 466                         pr_info("vb2:     wait_prepare: %u wait_finish: %u\n",
 467                                 q->cnt_wait_prepare, q->cnt_wait_finish);
 468                 }
 469                 q->cnt_queue_setup = 0;
 470                 q->cnt_wait_prepare = 0;
 471                 q->cnt_wait_finish = 0;
 472                 q->cnt_start_streaming = 0;
 473                 q->cnt_stop_streaming = 0;
 474         }
 475         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
 476                 struct vb2_buffer *vb = q->bufs[buffer];
 477                 bool unbalanced = vb->cnt_mem_alloc != vb->cnt_mem_put ||
 478                                   vb->cnt_mem_prepare != vb->cnt_mem_finish ||
 479                                   vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr ||
 480                                   vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf ||
 481                                   vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf ||
 482                                   vb->cnt_buf_queue != vb->cnt_buf_done ||
 483                                   vb->cnt_buf_prepare != vb->cnt_buf_finish ||
 484                                   vb->cnt_buf_init != vb->cnt_buf_cleanup;
 485
 486                 if (unbalanced || debug) {
 487                         pr_info("vb2:   counters for queue %p, buffer %d:%s\n",
 488                                 q, buffer, unbalanced ? " UNBALANCED!" : "");
 489                         pr_info("vb2:     buf_init: %u buf_cleanup: %u buf_prepare: %u buf_finish: %u\n",
 490                                 vb->cnt_buf_init, vb->cnt_buf_cleanup,
 491                                 vb->cnt_buf_prepare, vb->cnt_buf_finish);
 492                         pr_info("vb2:     buf_queue: %u buf_done: %u\n",
 493                                 vb->cnt_buf_queue, vb->cnt_buf_done);
 494                         pr_info("vb2:     alloc: %u put: %u prepare: %u finish: %u mmap: %u\n",
 495                                 vb->cnt_mem_alloc, vb->cnt_mem_put,
 496                                 vb->cnt_mem_prepare, vb->cnt_mem_finish,
 497                                 vb->cnt_mem_mmap);
 498                         pr_info("vb2:     get_userptr: %u put_userptr: %u\n",
 499                                 vb->cnt_mem_get_userptr, vb->cnt_mem_put_userptr);
 500                         pr_info("vb2:     attach_dmabuf: %u detach_dmabuf: %u map_dmabuf: %u unmap_dmabuf: %u\n",
 501                                 vb->cnt_mem_attach_dmabuf, vb->cnt_mem_detach_dmabuf,
 502                                 vb->cnt_mem_map_dmabuf, vb->cnt_mem_unmap_dmabuf);
 503                         pr_info("vb2:     get_dmabuf: %u num_users: %u vaddr: %u cookie: %u\n",
 504                                 vb->cnt_mem_get_dmabuf,
 505                                 vb->cnt_mem_num_users,
 506                                 vb->cnt_mem_vaddr,
 507                                 vb->cnt_mem_cookie);
 508                 }
 509         }
 510 #endif
 511
 512         /* Free videobuf buffers */
 513         for (buffer = q->num_buffers - buffers; buffer < q->num_buffers;
 514              ++buffer) {
 515                 kfree(q->bufs[buffer]);
 516                 q->bufs[buffer] = NULL;
 517         }
 518
 519         q->num_buffers -= buffers;
 520         if (!q->num_buffers) {
 521                 q->memory = 0;
 522                 INIT_LIST_HEAD(&q->queued_list);
 523         }
 524         return 0;
 525 }
 526
 527 /**
 528  * vb2_buffer_in_use() - return true if the buffer is in use and
 529  * the queue cannot be freed (by the means of REQBUFS(0)) call
 530  */
 531 bool vb2_buffer_in_use(struct vb2_queue *q, struct vb2_buffer *vb)
 532 {
 533         unsigned int plane;
 534         for (plane = 0; plane < vb->num_planes; ++plane) {
 535                 void *mem_priv = vb->planes[plane].mem_priv;
 536                 /*
 537                  * If num_users() has not been provided, call_memop
 538                  * will return 0, apparently nobody cares about this
 539                  * case anyway. If num_users() returns more than 1,
 540                  * we are not the only user of the plane's memory.
 541                  */
 542                 if (mem_priv && call_memop(vb, num_users, mem_priv) > 1)
 543                         return true;
 544         }
 545         return false;
 546 }
 547 EXPORT_SYMBOL(vb2_buffer_in_use);
 548
 549 /**
 550  * __buffers_in_use() - return true if any buffers on the queue are in use and
 551  * the queue cannot be freed (by the means of REQBUFS(0)) call
 552  */
 553 static bool __buffers_in_use(struct vb2_queue *q)
 554 {
 555         unsigned int buffer;
 556         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
 557                 if (vb2_buffer_in_use(q, q->bufs[buffer]))
 558                         return true;
 559         }
 560         return false;
 561 }
 562
 563 /**
 564  * vb2_core_querybuf() - query video buffer information
 565  * @q:          videobuf queue
 566  * @index:      id number of the buffer
 567  * @pb:         buffer struct passed from userspace
 568  *
 569  * Should be called from vidioc_querybuf ioctl handler in driver.
 570  * The passed buffer should have been verified.
 571  * This function fills the relevant information for the userspace.
 572  */
 573 void vb2_core_querybuf(struct vb2_queue *q, unsigned int index, void *pb)
 574 {
 575         call_void_bufop(q, fill_user_buffer, q->bufs[index], pb);
 576 }
 577 EXPORT_SYMBOL_GPL(vb2_core_querybuf);
 578
 579 /**
 580  * __verify_userptr_ops() - verify that all memory operations required for
 581  * USERPTR queue type have been provided
 582  */
 583 static int __verify_userptr_ops(struct vb2_queue *q)
 584 {
 585         if (!(q->io_modes & VB2_USERPTR) || !q->mem_ops->get_userptr ||
 586             !q->mem_ops->put_userptr)
 587                 return -EINVAL;
 588
 589         return 0;
 590 }
 591
 592 /**
 593  * __verify_mmap_ops() - verify that all memory operations required for
 594  * MMAP queue type have been provided
 595  */
 596 static int __verify_mmap_ops(struct vb2_queue *q)
 597 {
 598         if (!(q->io_modes & VB2_MMAP) || !q->mem_ops->alloc ||
 599             !q->mem_ops->put || !q->mem_ops->mmap)
 600                 return -EINVAL;
 601
 602         return 0;
 603 }
 604
 605 /**
 606  * __verify_dmabuf_ops() - verify that all memory operations required for
 607  * DMABUF queue type have been provided
 608  */
 609 static int __verify_dmabuf_ops(struct vb2_queue *q)
 610 {
 611         if (!(q->io_modes & VB2_DMABUF) || !q->mem_ops->attach_dmabuf ||
 612             !q->mem_ops->detach_dmabuf  || !q->mem_ops->map_dmabuf ||
 613             !q->mem_ops->unmap_dmabuf)
 614                 return -EINVAL;
 615
 616         return 0;
 617 }
 618
 619 /**
 620  * vb2_verify_memory_type() - Check whether the memory type and buffer type
 621  * passed to a buffer operation are compatible with the queue.
 622  */
 623 int vb2_verify_memory_type(struct vb2_queue *q,
 624                 enum vb2_memory memory, unsigned int type)
 625 {
 626         if (memory != VB2_MEMORY_MMAP && memory != VB2_MEMORY_USERPTR &&
 627             memory != VB2_MEMORY_DMABUF) {
 628                 dprintk(1, "unsupported memory type\n");
 629                 return -EINVAL;
 630         }
 631
 632         if (type != q->type) {
 633                 dprintk(1, "requested type is incorrect\n");
 634                 return -EINVAL;
 635         }
 636
 637         /*
 638          * Make sure all the required memory ops for given memory type
 639          * are available.
 640          */
 641         if (memory == VB2_MEMORY_MMAP && __verify_mmap_ops(q)) {
 642                 dprintk(1, "MMAP for current setup unsupported\n");
 643                 return -EINVAL;
 644         }
 645
 646         if (memory == VB2_MEMORY_USERPTR && __verify_userptr_ops(q)) {
 647                 dprintk(1, "USERPTR for current setup unsupported\n");
 648                 return -EINVAL;
 649         }
 650
 651         if (memory == VB2_MEMORY_DMABUF && __verify_dmabuf_ops(q)) {
 652                 dprintk(1, "DMABUF for current setup unsupported\n");
 653                 return -EINVAL;
 654         }
 655
 656         /*
 657          * Place the busy tests at the end: -EBUSY can be ignored when
 658          * create_bufs is called with count == 0, but count == 0 should still
 659          * do the memory and type validation.
 660          */
 661         if (vb2_fileio_is_active(q)) {
 662                 dprintk(1, "file io in progress\n");
 663                 return -EBUSY;
 664         }
 665         return 0;
 666 }
 667 EXPORT_SYMBOL(vb2_verify_memory_type);
 668
 669 /**
 670  * vb2_core_reqbufs() - Initiate streaming
 671  * @q:          videobuf2 queue
 672  * @memory: memory type
 673  * @count: requested buffer count
 674  *
 675  * Should be called from vidioc_reqbufs ioctl handler of a driver.
 676  * This function:
 677  * 1) verifies streaming parameters passed from the userspace,
 678  * 2) sets up the queue,
 679  * 3) negotiates number of buffers and planes per buffer with the driver
 680  *    to be used during streaming,
 681  * 4) allocates internal buffer structures (struct vb2_buffer), according to
 682  *    the agreed parameters,
 683  * 5) for MMAP memory type, allocates actual video memory, using the
 684  *    memory handling/allocation routines provided during queue initialization
 685  *
 686  * If req->count is 0, all the memory will be freed instead.
 687  * If the queue has been allocated previously (by a previous vb2_reqbufs) call
 688  * and the queue is not busy, memory will be reallocated.
 689  *
 690  * The return values from this function are intended to be directly returned
 691  * from vidioc_reqbufs handler in driver.
 692  */
 693 int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 694                 unsigned int *count)
 695 {
 696         unsigned int num_buffers, allocated_buffers, num_planes = 0;
 697         unsigned plane_sizes[VB2_MAX_PLANES] = { };
 698         int ret;
 699
 700         if (q->streaming) {
 701                 dprintk(1, "streaming active\n");
 702                 return -EBUSY;
 703         }
 704
 705         if (*count == 0 || q->num_buffers != 0 || q->memory != memory) {
 706                 /*
 707                  * We already have buffers allocated, so first check if they
 708                  * are not in use and can be freed.
 709                  */
 710                 mutex_lock(&q->mmap_lock);
 711                 if (q->memory == VB2_MEMORY_MMAP && __buffers_in_use(q)) {
 712                         mutex_unlock(&q->mmap_lock);
 713                         dprintk(1, "memory in use, cannot free\n");
 714                         return -EBUSY;
 715                 }
 716
 717                 /*
 718                  * Call queue_cancel to clean up any buffers in the PREPARED or
 719                  * QUEUED state which is possible if buffers were prepared or
 720                  * queued without ever calling STREAMON.
 721                  */
 722                 __vb2_queue_cancel(q);
 723                 ret = __vb2_queue_free(q, q->num_buffers);
 724                 mutex_unlock(&q->mmap_lock);
 725                 if (ret)
 726                         return ret;
 727
 728                 /*
 729                  * In case of REQBUFS(0) return immediately without calling
 730                  * driver's queue_setup() callback and allocating resources.
 731                  */
 732                 if (*count == 0)
 733                         return 0;
 734         }
 735
 736         /*
 737          * Make sure the requested values and current defaults are sane.
 738          */
 739         num_buffers = min_t(unsigned int, *count, VB2_MAX_FRAME);
 740         num_buffers = max_t(unsigned int, num_buffers, q->min_buffers_needed);
 741         memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 742         q->memory = memory;
 743
 744         /*
 745          * Ask the driver how many buffers and planes per buffer it requires.
 746          * Driver also sets the size and allocator context for each plane.
 747          */
 748         ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes,
 749                        plane_sizes, q->alloc_devs);
 750         if (ret)
 751                 return ret;
 752
 753         /* Finally, allocate buffers and video memory */
 754         allocated_buffers =
 755                 __vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes);
 756         if (allocated_buffers == 0) {
 757                 dprintk(1, "memory allocation failed\n");
 758                 return -ENOMEM;
 759         }
 760
 761         /*
 762          * There is no point in continuing if we can't allocate the minimum
 763          * number of buffers needed by this vb2_queue.
 764          */
 765         if (allocated_buffers < q->min_buffers_needed)
 766                 ret = -ENOMEM;
 767
 768         /*
 769          * Check if driver can handle the allocated number of buffers.
 770          */
 771         if (!ret && allocated_buffers < num_buffers) {
 772                 num_buffers = allocated_buffers;
 773                 /*
 774                  * num_planes is set by the previous queue_setup(), but since it
 775                  * signals to queue_setup() whether it is called from create_bufs()
 776                  * vs reqbufs() we zero it here to signal that queue_setup() is
 777                  * called for the reqbufs() case.
 778                  */
 779                 num_planes = 0;
 780
 781                 ret = call_qop(q, queue_setup, q, &num_buffers,
 782                                &num_planes, plane_sizes, q->alloc_devs);
 783
 784                 if (!ret && allocated_buffers < num_buffers)
 785                         ret = -ENOMEM;
 786
 787                 /*
 788                  * Either the driver has accepted a smaller number of buffers,
 789                  * or .queue_setup() returned an error
 790                  */
 791         }
 792
 793         mutex_lock(&q->mmap_lock);
 794         q->num_buffers = allocated_buffers;
 795
 796         if (ret < 0) {
 797                 /*
 798                  * Note: __vb2_queue_free() will subtract 'allocated_buffers'
 799                  * from q->num_buffers.
 800                  */
 801                 __vb2_queue_free(q, allocated_buffers);
 802                 mutex_unlock(&q->mmap_lock);
 803                 return ret;
 804         }
 805         mutex_unlock(&q->mmap_lock);
 806
 807         /*
 808          * Return the number of successfully allocated buffers
 809          * to the userspace.
 810          */
 811         *count = allocated_buffers;
 812         q->waiting_for_buffers = !q->is_output;
 813
 814         return 0;
 815 }
 816 EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
 817
 818 /**
 819  * vb2_core_create_bufs() - Allocate buffers and any required auxiliary structs
 820  * @q:          videobuf2 queue
 821  * @memory: memory type
 822  * @count: requested buffer count
 823  * @parg: parameter passed to device driver
 824  *
 825  * Should be called from vidioc_create_bufs ioctl handler of a driver.
 826  * This function:
 827  * 1) verifies parameter sanity
 828  * 2) calls the .queue_setup() queue operation
 829  * 3) performs any necessary memory allocations
 830  *
 831  * The return values from this function are intended to be directly returned
 832  * from vidioc_create_bufs handler in driver.
 833  */
 834 int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
 835                 unsigned int *count, unsigned requested_planes,
 836                 const unsigned requested_sizes[])
 837 {
 838         unsigned int num_planes = 0, num_buffers, allocated_buffers;
 839         unsigned plane_sizes[VB2_MAX_PLANES] = { };
 840         int ret;
 841
 842         if (q->num_buffers == VB2_MAX_FRAME) {
 843                 dprintk(1, "maximum number of buffers already allocated\n");
 844                 return -ENOBUFS;
 845         }
 846
 847         if (!q->num_buffers) {
 848                 memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 849                 q->memory = memory;
 850                 q->waiting_for_buffers = !q->is_output;
 851         }
 852
 853         num_buffers = min(*count, VB2_MAX_FRAME - q->num_buffers);
 854
 855         if (requested_planes && requested_sizes) {
 856                 num_planes = requested_planes;
 857                 memcpy(plane_sizes, requested_sizes, sizeof(plane_sizes));
 858         }
 859
 860         /*
 861          * Ask the driver, whether the requested number of buffers, planes per
 862          * buffer and their sizes are acceptable
 863          */
 864         ret = call_qop(q, queue_setup, q, &num_buffers,
 865                        &num_planes, plane_sizes, q->alloc_devs);
 866         if (ret)
 867                 return ret;
 868
 869         /* Finally, allocate buffers and video memory */
 870         allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers,
 871                                 num_planes, plane_sizes);
 872         if (allocated_buffers == 0) {
 873                 dprintk(1, "memory allocation failed\n");
 874                 return -ENOMEM;
 875         }
 876
 877         /*
 878          * Check if driver can handle the so far allocated number of buffers.
 879          */
 880         if (allocated_buffers < num_buffers) {
 881                 num_buffers = allocated_buffers;
 882
 883                 /*
 884                  * q->num_buffers contains the total number of buffers, that the
 885                  * queue driver has set up
 886                  */
 887                 ret = call_qop(q, queue_setup, q, &num_buffers,
 888                                &num_planes, plane_sizes, q->alloc_devs);
 889
 890                 if (!ret && allocated_buffers < num_buffers)
 891                         ret = -ENOMEM;
 892
 893                 /*
 894                  * Either the driver has accepted a smaller number of buffers,
 895                  * or .queue_setup() returned an error
 896                  */
 897         }
 898
 899         mutex_lock(&q->mmap_lock);
 900         q->num_buffers += allocated_buffers;
 901
 902         if (ret < 0) {
 903                 /*
 904                  * Note: __vb2_queue_free() will subtract 'allocated_buffers'
 905                  * from q->num_buffers.
 906                  */
 907                 __vb2_queue_free(q, allocated_buffers);
 908                 mutex_unlock(&q->mmap_lock);
 909                 return -ENOMEM;
 910         }
 911         mutex_unlock(&q->mmap_lock);
 912
 913         /*
 914          * Return the number of successfully allocated buffers
 915          * to the userspace.
 916          */
 917         *count = allocated_buffers;
 918
 919         return 0;
 920 }
 921 EXPORT_SYMBOL_GPL(vb2_core_create_bufs);
 922
 923 /**
 924  * vb2_plane_vaddr() - Return a kernel virtual address of a given plane
 925  * @vb:         vb2_buffer to which the plane in question belongs to
 926  * @plane_no:   plane number for which the address is to be returned
 927  *
 928  * This function returns a kernel virtual address of a given plane if
 929  * such a mapping exist, NULL otherwise.
 930  */
 931 void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
 932 {
 933         if (plane_no > vb->num_planes || !vb->planes[plane_no].mem_priv)
 934                 return NULL;
 935
 936         return call_ptr_memop(vb, vaddr, vb->planes[plane_no].mem_priv);
 937
 938 }
 939 EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
 940
 941 /**
 942  * vb2_plane_cookie() - Return allocator specific cookie for the given plane
 943  * @vb:         vb2_buffer to which the plane in question belongs to
 944  * @plane_no:   plane number for which the cookie is to be returned
 945  *
 946  * This function returns an allocator specific cookie for a given plane if
 947  * available, NULL otherwise. The allocator should provide some simple static
 948  * inline function, which would convert this cookie to the allocator specific
 949  * type that can be used directly by the driver to access the buffer. This can
 950  * be for example physical address, pointer to scatter list or IOMMU mapping.
 951  */
 952 void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
 953 {
 954         if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
 955                 return NULL;
 956
 957         return call_ptr_memop(vb, cookie, vb->planes[plane_no].mem_priv);
 958 }
 959 EXPORT_SYMBOL_GPL(vb2_plane_cookie);
 960
 961 /**
 962  * vb2_buffer_done() - inform videobuf that an operation on a buffer is finished
 963  * @vb:         vb2_buffer returned from the driver
 964  * @state:      either VB2_BUF_STATE_DONE if the operation finished successfully,
 965  *              VB2_BUF_STATE_ERROR if the operation finished with an error or
 966  *              VB2_BUF_STATE_QUEUED if the driver wants to requeue buffers.
 967  *              If start_streaming fails then it should return buffers with state
 968  *              VB2_BUF_STATE_QUEUED to put them back into the queue.
 969  *
 970  * This function should be called by the driver after a hardware operation on
 971  * a buffer is finished and the buffer may be returned to userspace. The driver
 972  * cannot use this buffer anymore until it is queued back to it by videobuf
 973  * by the means of buf_queue callback. Only buffers previously queued to the
 974  * driver by buf_queue can be passed to this function.
 975  *
 976  * While streaming a buffer can only be returned in state DONE or ERROR.
 977  * The start_streaming op can also return them in case the DMA engine cannot
 978  * be started for some reason. In that case the buffers should be returned with
 979  * state QUEUED.
 980  */
 981 void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 982 {
 983         struct vb2_queue *q = vb->vb2_queue;
 984         unsigned long flags;
 985         unsigned int plane;
 986
 987         if (WARN_ON(vb->state != VB2_BUF_STATE_ACTIVE))
 988                 return;
 989
 990         if (WARN_ON(state != VB2_BUF_STATE_DONE &&
 991                     state != VB2_BUF_STATE_ERROR &&
 992                     state != VB2_BUF_STATE_QUEUED &&
 993                     state != VB2_BUF_STATE_REQUEUEING))
 994                 state = VB2_BUF_STATE_ERROR;
 995
 996 #ifdef CONFIG_VIDEO_ADV_DEBUG
 997         /*
 998          * Although this is not a callback, it still does have to balance
 999          * with the buf_queue op. So update this counter manually.
1000          */
1001         vb->cnt_buf_done++;
1002 #endif
1003         dprintk(4, "done processing on buffer %d, state: %d\n",
1004                         vb->index, state);
1005
1006         /* sync buffers */
1007         for (plane = 0; plane < vb->num_planes; ++plane)
1008                 call_void_memop(vb, finish, vb->planes[plane].mem_priv);
1009
1010         spin_lock_irqsave(&q->done_lock, flags);
1011         if (state == VB2_BUF_STATE_QUEUED ||
1012             state == VB2_BUF_STATE_REQUEUEING) {
1013                 vb->state = VB2_BUF_STATE_QUEUED;
1014         } else {
1015                 /* Add the buffer to the done buffers list */
1016                 list_add_tail(&vb->done_entry, &q->done_list);
1017                 vb->state = state;
1018         }
1019         atomic_dec(&q->owned_by_drv_count);
1020         spin_unlock_irqrestore(&q->done_lock, flags);
1021
1022         trace_vb2_buf_done(q, vb);
1023
1024         switch (state) {
1025         case VB2_BUF_STATE_QUEUED:
1026                 return;
1027         case VB2_BUF_STATE_REQUEUEING:
1028                 if (q->start_streaming_called)
1029                         __enqueue_in_driver(vb);
1030                 return;
1031         default:
1032                 /* Inform any processes that may be waiting for buffers */
1033                 wake_up(&q->done_wq);
1034                 break;
1035         }
1036 }
1037 EXPORT_SYMBOL_GPL(vb2_buffer_done);
1038
1039 /**
1040  * vb2_discard_done() - discard all buffers marked as DONE
1041  * @q:          videobuf2 queue
1042  *
1043  * This function is intended to be used with suspend/resume operations. It
1044  * discards all 'done' buffers as they would be too old to be requested after
1045  * resume.
1046  *
1047  * Drivers must stop the hardware and synchronize with interrupt handlers and/or
1048  * delayed works before calling this function to make sure no buffer will be
1049  * touched by the driver and/or hardware.
1050  */
1051 void vb2_discard_done(struct vb2_queue *q)
1052 {
1053         struct vb2_buffer *vb;
1054         unsigned long flags;
1055
1056         spin_lock_irqsave(&q->done_lock, flags);
1057         list_for_each_entry(vb, &q->done_list, done_entry)
1058                 vb->state = VB2_BUF_STATE_ERROR;
1059         spin_unlock_irqrestore(&q->done_lock, flags);
1060 }
1061 EXPORT_SYMBOL_GPL(vb2_discard_done);
1062
1063 /**
1064  * __qbuf_mmap() - handle qbuf of an MMAP buffer
1065  */
1066 static int __qbuf_mmap(struct vb2_buffer *vb, const void *pb)
1067 {
1068         int ret = 0;
1069
1070         if (pb)
1071                 ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1072                                  vb, pb, vb->planes);
1073         return ret ? ret : call_vb_qop(vb, buf_prepare, vb);
1074 }
1075
1076 /**
1077  * __qbuf_userptr() - handle qbuf of a USERPTR buffer
1078  */
1079 static int __qbuf_userptr(struct vb2_buffer *vb, const void *pb)
1080 {
1081         struct vb2_plane planes[VB2_MAX_PLANES];
1082         struct vb2_queue *q = vb->vb2_queue;
1083         void *mem_priv;
1084         unsigned int plane;
1085         int ret = 0;
1086         enum dma_data_direction dma_dir =
1087                 q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
1088         bool reacquired = vb->planes[0].mem_priv == NULL;
1089
1090         memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1091         /* Copy relevant information provided by the userspace */
1092         if (pb)
1093                 ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1094                                  vb, pb, planes);
1095         if (ret)
1096                 return ret;
1097
1098         for (plane = 0; plane < vb->num_planes; ++plane) {
1099                 /* Skip the plane if already verified */
1100                 if (vb->planes[plane].m.userptr &&
1101                         vb->planes[plane].m.userptr == planes[plane].m.userptr
1102                         && vb->planes[plane].length == planes[plane].length)
1103                         continue;
1104
1105                 dprintk(3, "userspace address for plane %d changed, "
1106                                 "reacquiring memory\n", plane);
1107
1108                 /* Check if the provided plane buffer is large enough */
1109                 if (planes[plane].length < vb->planes[plane].min_length) {
1110                         dprintk(1, "provided buffer size %u is less than "
1111                                                 "setup size %u for plane %d\n",
1112                                                 planes[plane].length,
1113                                                 vb->planes[plane].min_length,
1114                                                 plane);
1115                         ret = -EINVAL;
1116                         goto err;
1117                 }
1118
1119                 /* Release previously acquired memory if present */
1120                 if (vb->planes[plane].mem_priv) {
1121                         if (!reacquired) {
1122                                 reacquired = true;
1123                                 call_void_vb_qop(vb, buf_cleanup, vb);
1124                         }
1125                         call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
1126                 }
1127
1128                 vb->planes[plane].mem_priv = NULL;
1129                 vb->planes[plane].bytesused = 0;
1130                 vb->planes[plane].length = 0;
1131                 vb->planes[plane].m.userptr = 0;
1132                 vb->planes[plane].data_offset = 0;
1133
1134                 /* Acquire each plane's memory */
1135                 mem_priv = call_ptr_memop(vb, get_userptr,
1136                                 q->alloc_devs[plane] ? : q->dev,
1137                                 planes[plane].m.userptr,
1138                                 planes[plane].length, dma_dir);
1139                 if (IS_ERR_OR_NULL(mem_priv)) {
1140                         dprintk(1, "failed acquiring userspace "
1141                                                 "memory for plane %d\n", plane);
1142                         ret = mem_priv ? PTR_ERR(mem_priv) : -EINVAL;
1143                         goto err;
1144                 }
1145                 vb->planes[plane].mem_priv = mem_priv;
1146         }
1147
1148         /*
1149          * Now that everything is in order, copy relevant information
1150          * provided by userspace.
1151          */
1152         for (plane = 0; plane < vb->num_planes; ++plane) {
1153                 vb->planes[plane].bytesused = planes[plane].bytesused;
1154                 vb->planes[plane].length = planes[plane].length;
1155                 vb->planes[plane].m.userptr = planes[plane].m.userptr;
1156                 vb->planes[plane].data_offset = planes[plane].data_offset;
1157         }
1158
1159         if (reacquired) {
1160                 /*
1161                  * One or more planes changed, so we must call buf_init to do
1162                  * the driver-specific initialization on the newly acquired
1163                  * buffer, if provided.
1164                  */
1165                 ret = call_vb_qop(vb, buf_init, vb);
1166                 if (ret) {
1167                         dprintk(1, "buffer initialization failed\n");
1168                         goto err;
1169                 }
1170         }
1171
1172         ret = call_vb_qop(vb, buf_prepare, vb);
1173         if (ret) {
1174                 dprintk(1, "buffer preparation failed\n");
1175                 call_void_vb_qop(vb, buf_cleanup, vb);
1176                 goto err;
1177         }
1178
1179         return 0;
1180 err:
1181         /* In case of errors, release planes that were already acquired */
1182         for (plane = 0; plane < vb->num_planes; ++plane) {
1183                 if (vb->planes[plane].mem_priv)
1184                         call_void_memop(vb, put_userptr,
1185                                 vb->planes[plane].mem_priv);
1186                 vb->planes[plane].mem_priv = NULL;
1187                 vb->planes[plane].m.userptr = 0;
1188                 vb->planes[plane].length = 0;
1189         }
1190
1191         return ret;
1192 }
1193
1194 /**
1195  * __qbuf_dmabuf() - handle qbuf of a DMABUF buffer
1196  */
1197 static int __qbuf_dmabuf(struct vb2_buffer *vb, const void *pb)
1198 {
1199         struct vb2_plane planes[VB2_MAX_PLANES];
1200         struct vb2_queue *q = vb->vb2_queue;
1201         void *mem_priv;
1202         unsigned int plane;
1203         int ret = 0;
1204         enum dma_data_direction dma_dir =
1205                 q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
1206         bool reacquired = vb->planes[0].mem_priv == NULL;
1207
1208         memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1209         /* Copy relevant information provided by the userspace */
1210         if (pb)
1211                 ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1212                                  vb, pb, planes);
1213         if (ret)
1214                 return ret;
1215
1216         for (plane = 0; plane < vb->num_planes; ++plane) {
1217                 struct dma_buf *dbuf = dma_buf_get(planes[plane].m.fd);
1218
1219                 if (IS_ERR_OR_NULL(dbuf)) {
1220                         dprintk(1, "invalid dmabuf fd for plane %d\n",
1221                                 plane);
1222                         ret = -EINVAL;
1223                         goto err;
1224                 }
1225
1226                 /* use DMABUF size if length is not provided */
1227                 if (planes[plane].length == 0)
1228                         planes[plane].length = dbuf->size;
1229
1230                 if (planes[plane].length < vb->planes[plane].min_length) {
1231                         dprintk(1, "invalid dmabuf length for plane %d\n",
1232                                 plane);
1233                         dma_buf_put(dbuf);
1234                         ret = -EINVAL;
1235                         goto err;
1236                 }
1237
1238                 /* Skip the plane if already verified */
1239                 if (dbuf == vb->planes[plane].dbuf &&
1240                         vb->planes[plane].length == planes[plane].length) {
1241                         dma_buf_put(dbuf);
1242                         continue;
1243                 }
1244
1245                 dprintk(1, "buffer for plane %d changed\n", plane);
1246
1247                 if (!reacquired) {
1248                         reacquired = true;
1249                         call_void_vb_qop(vb, buf_cleanup, vb);
1250                 }
1251
1252                 /* Release previously acquired memory if present */
1253                 __vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
1254                 vb->planes[plane].bytesused = 0;
1255                 vb->planes[plane].length = 0;
1256                 vb->planes[plane].m.fd = 0;
1257                 vb->planes[plane].data_offset = 0;
1258
1259                 /* Acquire each plane's memory */
1260                 mem_priv = call_ptr_memop(vb, attach_dmabuf,
1261                                 q->alloc_devs[plane] ? : q->dev,
1262                                 dbuf, planes[plane].length, dma_dir);
1263                 if (IS_ERR(mem_priv)) {
1264                         dprintk(1, "failed to attach dmabuf\n");
1265                         ret = PTR_ERR(mem_priv);
1266                         dma_buf_put(dbuf);
1267                         goto err;
1268                 }
1269
1270                 vb->planes[plane].dbuf = dbuf;
1271                 vb->planes[plane].mem_priv = mem_priv;
1272         }
1273
1274         /* TODO: This pins the buffer(s) with  dma_buf_map_attachment()).. but
1275          * really we want to do this just before the DMA, not while queueing
1276          * the buffer(s)..
1277          */
1278         for (plane = 0; plane < vb->num_planes; ++plane) {
1279                 ret = call_memop(vb, map_dmabuf, vb->planes[plane].mem_priv);
1280                 if (ret) {
1281                         dprintk(1, "failed to map dmabuf for plane %d\n",
1282                                 plane);
1283                         goto err;
1284                 }
1285                 vb->planes[plane].dbuf_mapped = 1;
1286         }
1287
1288         /*
1289          * Now that everything is in order, copy relevant information
1290          * provided by userspace.
1291          */
1292         for (plane = 0; plane < vb->num_planes; ++plane) {
1293                 vb->planes[plane].bytesused = planes[plane].bytesused;
1294                 vb->planes[plane].length = planes[plane].length;
1295                 vb->planes[plane].m.fd = planes[plane].m.fd;
1296                 vb->planes[plane].data_offset = planes[plane].data_offset;
1297         }
1298
1299         if (reacquired) {
1300                 /*
1301                  * Call driver-specific initialization on the newly acquired buffer,
1302                  * if provided.
1303                  */
1304                 ret = call_vb_qop(vb, buf_init, vb);
1305                 if (ret) {
1306                         dprintk(1, "buffer initialization failed\n");
1307                         goto err;
1308                 }
1309         }
1310
1311         ret = call_vb_qop(vb, buf_prepare, vb);
1312         if (ret) {
1313                 dprintk(1, "buffer preparation failed\n");
1314                 call_void_vb_qop(vb, buf_cleanup, vb);
1315                 goto err;
1316         }
1317
1318         return 0;
1319 err:
1320         /* In case of errors, release planes that were already acquired */
1321         __vb2_buf_dmabuf_put(vb);
1322
1323         return ret;
1324 }
1325
1326 /**
1327  * __enqueue_in_driver() - enqueue a vb2_buffer in driver for processing
1328  */
1329 static void __enqueue_in_driver(struct vb2_buffer *vb)
1330 {
1331         struct vb2_queue *q = vb->vb2_queue;
1332         unsigned int plane;
1333
1334         vb->state = VB2_BUF_STATE_ACTIVE;
1335         atomic_inc(&q->owned_by_drv_count);
1336
1337         trace_vb2_buf_queue(q, vb);
1338
1339         /* sync buffers */
1340         for (plane = 0; plane < vb->num_planes; ++plane)
1341                 call_void_memop(vb, prepare, vb->planes[plane].mem_priv);
1342
1343         call_void_vb_qop(vb, buf_queue, vb);
1344 }
1345
1346 static int __buf_prepare(struct vb2_buffer *vb, const void *pb)
1347 {
1348         struct vb2_queue *q = vb->vb2_queue;
1349         int ret;
1350
1351         if (q->error) {
1352                 dprintk(1, "fatal error occurred on queue\n");
1353                 return -EIO;
1354         }
1355
1356         vb->state = VB2_BUF_STATE_PREPARING;
1357
1358         switch (q->memory) {
1359         case VB2_MEMORY_MMAP:
1360                 ret = __qbuf_mmap(vb, pb);
1361                 break;
1362         case VB2_MEMORY_USERPTR:
1363                 ret = __qbuf_userptr(vb, pb);
1364                 break;
1365         case VB2_MEMORY_DMABUF:
1366                 ret = __qbuf_dmabuf(vb, pb);
1367                 break;
1368         default:
1369                 WARN(1, "Invalid queue type\n");
1370                 ret = -EINVAL;
1371         }
1372
1373         if (ret)
1374                 dprintk(1, "buffer preparation failed: %d\n", ret);
1375         vb->state = ret ? VB2_BUF_STATE_DEQUEUED : VB2_BUF_STATE_PREPARED;
1376
1377         return ret;
1378 }
1379
1380 /**
1381  * vb2_core_prepare_buf() - Pass ownership of a buffer from userspace
1382  *                      to the kernel
1383  * @q:          videobuf2 queue
1384  * @index:      id number of the buffer
1385  * @pb:         buffer structure passed from userspace to vidioc_prepare_buf
1386  *              handler in driver
1387  *
1388  * Should be called from vidioc_prepare_buf ioctl handler of a driver.
1389  * The passed buffer should have been verified.
1390  * This function calls buf_prepare callback in the driver (if provided),
1391  * in which driver-specific buffer initialization can be performed,
1392  *
1393  * The return values from this function are intended to be directly returned
1394  * from vidioc_prepare_buf handler in driver.
1395  */
1396 int vb2_core_prepare_buf(struct vb2_queue *q, unsigned int index, void *pb)
1397 {
1398         struct vb2_buffer *vb;
1399         int ret;
1400
1401         vb = q->bufs[index];
1402         if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1403                 dprintk(1, "invalid buffer state %d\n",
1404                         vb->state);
1405                 return -EINVAL;
1406         }
1407
1408         ret = __buf_prepare(vb, pb);
1409         if (ret)
1410                 return ret;
1411
1412         /* Fill buffer information for the userspace */
1413         call_void_bufop(q, fill_user_buffer, vb, pb);
1414
1415         dprintk(1, "prepare of buffer %d succeeded\n", vb->index);
1416
1417         return ret;
1418 }
1419 EXPORT_SYMBOL_GPL(vb2_core_prepare_buf);
1420
1421 /**
1422  * vb2_start_streaming() - Attempt to start streaming.
1423  * @q:          videobuf2 queue
1424  *
1425  * Attempt to start streaming. When this function is called there must be
1426  * at least q->min_buffers_needed buffers queued up (i.e. the minimum
1427  * number of buffers required for the DMA engine to function). If the
1428  * @start_streaming op fails it is supposed to return all the driver-owned
1429  * buffers back to vb2 in state QUEUED. Check if that happened and if
1430  * not warn and reclaim them forcefully.
1431  */
1432 static int vb2_start_streaming(struct vb2_queue *q)
1433 {
1434         struct vb2_buffer *vb;
1435         int ret;
1436
1437         /*
1438          * If any buffers were queued before streamon,
1439          * we can now pass them to driver for processing.
1440          */
1441         list_for_each_entry(vb, &q->queued_list, queued_entry)
1442                 __enqueue_in_driver(vb);
1443
1444         /* Tell the driver to start streaming */
1445         q->start_streaming_called = 1;
1446         ret = call_qop(q, start_streaming, q,
1447                        atomic_read(&q->owned_by_drv_count));
1448         if (!ret)
1449                 return 0;
1450
1451         q->start_streaming_called = 0;
1452
1453         dprintk(1, "driver refused to start streaming\n");
1454         /*
1455          * If you see this warning, then the driver isn't cleaning up properly
1456          * after a failed start_streaming(). See the start_streaming()
1457          * documentation in videobuf2-core.h for more information how buffers
1458          * should be returned to vb2 in start_streaming().
1459          */
1460         if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
1461                 unsigned i;
1462
1463                 /*
1464                  * Forcefully reclaim buffers if the driver did not
1465                  * correctly return them to vb2.
1466                  */
1467                 for (i = 0; i < q->num_buffers; ++i) {
1468                         vb = q->bufs[i];
1469                         if (vb->state == VB2_BUF_STATE_ACTIVE)
1470                                 vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED);
1471                 }
1472                 /* Must be zero now */
1473                 WARN_ON(atomic_read(&q->owned_by_drv_count));
1474         }
1475         /*
1476          * If done_list is not empty, then start_streaming() didn't call
1477          * vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED) but STATE_ERROR or
1478          * STATE_DONE.
1479          */
1480         WARN_ON(!list_empty(&q->done_list));
1481         return ret;
1482 }
1483
1484 /**
1485  * vb2_core_qbuf() - Queue a buffer from userspace
1486  * @q:          videobuf2 queue
1487  * @index:      id number of the buffer
1488  * @pb:         buffer structure passed from userspace to vidioc_qbuf handler
1489  *              in driver
1490  *
1491  * Should be called from vidioc_qbuf ioctl handler of a driver.
1492  * The passed buffer should have been verified.
1493  * This function:
1494  * 1) if necessary, calls buf_prepare callback in the driver (if provided), in
1495  *    which driver-specific buffer initialization can be performed,
1496  * 2) if streaming is on, queues the buffer in driver by the means of buf_queue
1497  *    callback for processing.
1498  *
1499  * The return values from this function are intended to be directly returned
1500  * from vidioc_qbuf handler in driver.
1501  */
1502 int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb)
1503 {
1504         struct vb2_buffer *vb;
1505         int ret;
1506
1507         vb = q->bufs[index];
1508
1509         switch (vb->state) {
1510         case VB2_BUF_STATE_DEQUEUED:
1511                 ret = __buf_prepare(vb, pb);
1512                 if (ret)
1513                         return ret;
1514                 break;
1515         case VB2_BUF_STATE_PREPARED:
1516                 break;
1517         case VB2_BUF_STATE_PREPARING:
1518                 dprintk(1, "buffer still being prepared\n");
1519                 return -EINVAL;
1520         default:
1521                 dprintk(1, "invalid buffer state %d\n", vb->state);
1522                 return -EINVAL;
1523         }
1524
1525         /*
1526          * Add to the queued buffers list, a buffer will stay on it until
1527          * dequeued in dqbuf.
1528          */
1529         list_add_tail(&vb->queued_entry, &q->queued_list);
1530         q->queued_count++;
1531         q->waiting_for_buffers = false;
1532         vb->state = VB2_BUF_STATE_QUEUED;
1533
1534         if (pb)
1535                 call_void_bufop(q, copy_timestamp, vb, pb);
1536
1537         trace_vb2_qbuf(q, vb);
1538
1539         /*
1540          * If already streaming, give the buffer to driver for processing.
1541          * If not, the buffer will be given to driver on next streamon.
1542          */
1543         if (q->start_streaming_called)
1544                 __enqueue_in_driver(vb);
1545
1546         /* Fill buffer information for the userspace */
1547         if (pb)
1548                 call_void_bufop(q, fill_user_buffer, vb, pb);
1549
1550         /*
1551          * If streamon has been called, and we haven't yet called
1552          * start_streaming() since not enough buffers were queued, and
1553          * we now have reached the minimum number of queued buffers,
1554          * then we can finally call start_streaming().
1555          */
1556         if (q->streaming && !q->start_streaming_called &&
1557             q->queued_count >= q->min_buffers_needed) {
1558                 ret = vb2_start_streaming(q);
1559                 if (ret)
1560                         return ret;
1561         }
1562
1563         dprintk(1, "qbuf of buffer %d succeeded\n", vb->index);
1564         return 0;
1565 }
1566 EXPORT_SYMBOL_GPL(vb2_core_qbuf);
1567
1568 /**
1569  * __vb2_wait_for_done_vb() - wait for a buffer to become available
1570  * for dequeuing
1571  *
1572  * Will sleep if required for nonblocking == false.
1573  */
1574 static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
1575 {
1576         /*
1577          * All operations on vb_done_list are performed under done_lock
1578          * spinlock protection. However, buffers may be removed from
1579          * it and returned to userspace only while holding both driver's
1580          * lock and the done_lock spinlock. Thus we can be sure that as
1581          * long as we hold the driver's lock, the list will remain not
1582          * empty if list_empty() check succeeds.
1583          */
1584
1585         for (;;) {
1586                 int ret;
1587
1588                 if (!q->streaming) {
1589                         dprintk(1, "streaming off, will not wait for buffers\n");
1590                         return -EINVAL;
1591                 }
1592
1593                 if (q->error) {
1594                         dprintk(1, "Queue in error state, will not wait for buffers\n");
1595                         return -EIO;
1596                 }
1597
1598                 if (q->last_buffer_dequeued) {
1599                         dprintk(3, "last buffer dequeued already, will not wait for buffers\n");
1600                         return -EPIPE;
1601                 }
1602
1603                 if (!list_empty(&q->done_list)) {
1604                         /*
1605                          * Found a buffer that we were waiting for.
1606                          */
1607                         break;
1608                 }
1609
1610                 if (nonblocking) {
1611                         dprintk(1, "nonblocking and no buffers to dequeue, "
1612                                                                 "will not wait\n");
1613                         return -EAGAIN;
1614                 }
1615
1616                 /*
1617                  * We are streaming and blocking, wait for another buffer to
1618                  * become ready or for streamoff. Driver's lock is released to
1619                  * allow streamoff or qbuf to be called while waiting.
1620                  */
1621                 call_void_qop(q, wait_prepare, q);
1622
1623                 /*
1624                  * All locks have been released, it is safe to sleep now.
1625                  */
1626                 dprintk(3, "will sleep waiting for buffers\n");
1627                 ret = wait_event_interruptible(q->done_wq,
1628                                 !list_empty(&q->done_list) || !q->streaming ||
1629                                 q->error);
1630
1631                 /*
1632                  * We need to reevaluate both conditions again after reacquiring
1633                  * the locks or return an error if one occurred.
1634                  */
1635                 call_void_qop(q, wait_finish, q);
1636                 if (ret) {
1637                         dprintk(1, "sleep was interrupted\n");
1638                         return ret;
1639                 }
1640         }
1641         return 0;
1642 }
1643
1644 /**
1645  * __vb2_get_done_vb() - get a buffer ready for dequeuing
1646  *
1647  * Will sleep if required for nonblocking == false.
1648  */
1649 static int __vb2_get_done_vb(struct vb2_queue *q, struct vb2_buffer **vb,
1650                              void *pb, int nonblocking)
1651 {
1652         unsigned long flags;
1653         int ret = 0;
1654
1655         /*
1656          * Wait for at least one buffer to become available on the done_list.
1657          */
1658         ret = __vb2_wait_for_done_vb(q, nonblocking);
1659         if (ret)
1660                 return ret;
1661
1662         /*
1663          * Driver's lock has been held since we last verified that done_list
1664          * is not empty, so no need for another list_empty(done_list) check.
1665          */
1666         spin_lock_irqsave(&q->done_lock, flags);
1667         *vb = list_first_entry(&q->done_list, struct vb2_buffer, done_entry);
1668         /*
1669          * Only remove the buffer from done_list if all planes can be
1670          * handled. Some cases such as V4L2 file I/O and DVB have pb
1671          * == NULL; skip the check then as there's nothing to verify.
1672          */
1673         if (pb)
1674                 ret = call_bufop(q, verify_planes_array, *vb, pb);
1675         if (!ret)
1676                 list_del(&(*vb)->done_entry);
1677         spin_unlock_irqrestore(&q->done_lock, flags);
1678
1679         return ret;
1680 }
1681
1682 /**
1683  * vb2_wait_for_all_buffers() - wait until all buffers are given back to vb2
1684  * @q:          videobuf2 queue
1685  *
1686  * This function will wait until all buffers that have been given to the driver
1687  * by buf_queue() are given back to vb2 with vb2_buffer_done(). It doesn't call
1688  * wait_prepare, wait_finish pair. It is intended to be called with all locks
1689  * taken, for example from stop_streaming() callback.
1690  */
1691 int vb2_wait_for_all_buffers(struct vb2_queue *q)
1692 {
1693         if (!q->streaming) {
1694                 dprintk(1, "streaming off, will not wait for buffers\n");
1695                 return -EINVAL;
1696         }
1697
1698         if (q->start_streaming_called)
1699                 wait_event(q->done_wq, !atomic_read(&q->owned_by_drv_count));
1700         return 0;
1701 }
1702 EXPORT_SYMBOL_GPL(vb2_wait_for_all_buffers);
1703
1704 /**
1705  * __vb2_dqbuf() - bring back the buffer to the DEQUEUED state
1706  */
1707 static void __vb2_dqbuf(struct vb2_buffer *vb)
1708 {
1709         struct vb2_queue *q = vb->vb2_queue;
1710         unsigned int i;
1711
1712         /* nothing to do if the buffer is already dequeued */
1713         if (vb->state == VB2_BUF_STATE_DEQUEUED)
1714                 return;
1715
1716         vb->state = VB2_BUF_STATE_DEQUEUED;
1717
1718         /* unmap DMABUF buffer */
1719         if (q->memory == VB2_MEMORY_DMABUF)
1720                 for (i = 0; i < vb->num_planes; ++i) {
1721                         if (!vb->planes[i].dbuf_mapped)
1722                                 continue;
1723                         call_void_memop(vb, unmap_dmabuf, vb->planes[i].mem_priv);
1724                         vb->planes[i].dbuf_mapped = 0;
1725                 }
1726 }
1727
1728 /**
1729  * vb2_dqbuf() - Dequeue a buffer to the userspace
1730  * @q:          videobuf2 queue
1731  * @pb:         buffer structure passed from userspace to vidioc_dqbuf handler
1732  *              in driver
1733  * @nonblocking: if true, this call will not sleep waiting for a buffer if no
1734  *               buffers ready for dequeuing are present. Normally the driver
1735  *               would be passing (file->f_flags & O_NONBLOCK) here
1736  *
1737  * Should be called from vidioc_dqbuf ioctl handler of a driver.
1738  * The passed buffer should have been verified.
1739  * This function:
1740  * 1) calls buf_finish callback in the driver (if provided), in which
1741  *    driver can perform any additional operations that may be required before
1742  *    returning the buffer to userspace, such as cache sync,
1743  * 2) the buffer struct members are filled with relevant information for
1744  *    the userspace.
1745  *
1746  * The return values from this function are intended to be directly returned
1747  * from vidioc_dqbuf handler in driver.
1748  */
1749 int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
1750                    bool nonblocking)
1751 {
1752         struct vb2_buffer *vb = NULL;
1753         int ret;
1754
1755         ret = __vb2_get_done_vb(q, &vb, pb, nonblocking);
1756         if (ret < 0)
1757                 return ret;
1758
1759         switch (vb->state) {
1760         case VB2_BUF_STATE_DONE:
1761                 dprintk(3, "returning done buffer\n");
1762                 break;
1763         case VB2_BUF_STATE_ERROR:
1764                 dprintk(3, "returning done buffer with errors\n");
1765                 break;
1766         default:
1767                 dprintk(1, "invalid buffer state\n");
1768                 return -EINVAL;
1769         }
1770
1771         call_void_vb_qop(vb, buf_finish, vb);
1772
1773         if (pindex)
1774                 *pindex = vb->index;
1775
1776         /* Fill buffer information for the userspace */
1777         if (pb)
1778                 call_void_bufop(q, fill_user_buffer, vb, pb);
1779
1780         /* Remove from videobuf queue */
1781         list_del(&vb->queued_entry);
1782         q->queued_count--;
1783
1784         trace_vb2_dqbuf(q, vb);
1785
1786         /* go back to dequeued state */
1787         __vb2_dqbuf(vb);
1788
1789         dprintk(1, "dqbuf of buffer %d, with state %d\n",
1790                         vb->index, vb->state);
1791
1792         return 0;
1793
1794 }
1795 EXPORT_SYMBOL_GPL(vb2_core_dqbuf);
1796
1797 /**
1798  * __vb2_queue_cancel() - cancel and stop (pause) streaming
1799  *
1800  * Removes all queued buffers from driver's queue and all buffers queued by
1801  * userspace from videobuf's queue. Returns to state after reqbufs.
1802  */
1803 static void __vb2_queue_cancel(struct vb2_queue *q)
1804 {
1805         unsigned int i;
1806
1807         /*
1808          * Tell driver to stop all transactions and release all queued
1809          * buffers.
1810          */
1811         if (q->start_streaming_called)
1812                 call_void_qop(q, stop_streaming, q);
1813
1814         /*
1815          * If you see this warning, then the driver isn't cleaning up properly
1816          * in stop_streaming(). See the stop_streaming() documentation in
1817          * videobuf2-core.h for more information how buffers should be returned
1818          * to vb2 in stop_streaming().
1819          */
1820         if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
1821                 for (i = 0; i < q->num_buffers; ++i)
1822                         if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE)
1823                                 vb2_buffer_done(q->bufs[i], VB2_BUF_STATE_ERROR);
1824                 /* Must be zero now */
1825                 WARN_ON(atomic_read(&q->owned_by_drv_count));
1826         }
1827
1828         q->streaming = 0;
1829         q->start_streaming_called = 0;
1830         q->queued_count = 0;
1831         q->error = 0;
1832
1833         /*
1834          * Remove all buffers from videobuf's list...
1835          */
1836         INIT_LIST_HEAD(&q->queued_list);
1837         /*
1838          * ...and done list; userspace will not receive any buffers it
1839          * has not already dequeued before initiating cancel.
1840          */
1841         INIT_LIST_HEAD(&q->done_list);
1842         atomic_set(&q->owned_by_drv_count, 0);
1843         wake_up_all(&q->done_wq);
1844
1845         /*
1846          * Reinitialize all buffers for next use.
1847          * Make sure to call buf_finish for any queued buffers. Normally
1848          * that's done in dqbuf, but that's not going to happen when we
1849          * cancel the whole queue. Note: this code belongs here, not in
1850          * __vb2_dqbuf() since in vb2_core_dqbuf() there is a critical
1851          * call to __fill_user_buffer() after buf_finish(). That order can't
1852          * be changed, so we can't move the buf_finish() to __vb2_dqbuf().
1853          */
1854         for (i = 0; i < q->num_buffers; ++i) {
1855                 struct vb2_buffer *vb = q->bufs[i];
1856
1857                 if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1858                         vb->state = VB2_BUF_STATE_PREPARED;
1859                         call_void_vb_qop(vb, buf_finish, vb);
1860                 }
1861                 __vb2_dqbuf(vb);
1862         }
1863 }
1864
1865 int vb2_core_streamon(struct vb2_queue *q, unsigned int type)
1866 {
1867         int ret;
1868
1869         if (type != q->type) {
1870                 dprintk(1, "invalid stream type\n");
1871                 return -EINVAL;
1872         }
1873
1874         if (q->streaming) {
1875                 dprintk(3, "already streaming\n");
1876                 return 0;
1877         }
1878
1879         if (!q->num_buffers) {
1880                 dprintk(1, "no buffers have been allocated\n");
1881                 return -EINVAL;
1882         }
1883
1884         if (q->num_buffers < q->min_buffers_needed) {
1885                 dprintk(1, "need at least %u allocated buffers\n",
1886                                 q->min_buffers_needed);
1887                 return -EINVAL;
1888         }
1889
1890         /*
1891          * Tell driver to start streaming provided sufficient buffers
1892          * are available.
1893          */
1894         if (q->queued_count >= q->min_buffers_needed) {
1895                 ret = v4l_vb2q_enable_media_source(q);
1896                 if (ret)
1897                         return ret;
1898                 ret = vb2_start_streaming(q);
1899                 if (ret) {
1900                         __vb2_queue_cancel(q);
1901                         return ret;
1902                 }
1903         }
1904
1905         q->streaming = 1;
1906
1907         dprintk(3, "successful\n");
1908         return 0;
1909 }
1910 EXPORT_SYMBOL_GPL(vb2_core_streamon);
1911
1912 /**
1913  * vb2_queue_error() - signal a fatal error on the queue
1914  * @q:          videobuf2 queue
1915  *
1916  * Flag that a fatal unrecoverable error has occurred and wake up all processes
1917  * waiting on the queue. Polling will now set POLLERR and queuing and dequeuing
1918  * buffers will return -EIO.
1919  *
1920  * The error flag will be cleared when cancelling the queue, either from
1921  * vb2_streamoff or vb2_queue_release. Drivers should thus not call this
1922  * function before starting the stream, otherwise the error flag will remain set
1923  * until the queue is released when closing the device node.
1924  */
1925 void vb2_queue_error(struct vb2_queue *q)
1926 {
1927         q->error = 1;
1928
1929         wake_up_all(&q->done_wq);
1930 }
1931 EXPORT_SYMBOL_GPL(vb2_queue_error);
1932
1933 int vb2_core_streamoff(struct vb2_queue *q, unsigned int type)
1934 {
1935         if (type != q->type) {
1936                 dprintk(1, "invalid stream type\n");
1937                 return -EINVAL;
1938         }
1939
1940         /*
1941          * Cancel will pause streaming and remove all buffers from the driver
1942          * and videobuf, effectively returning control over them to userspace.
1943          *
1944          * Note that we do this even if q->streaming == 0: if you prepare or
1945          * queue buffers, and then call streamoff without ever having called
1946          * streamon, you would still expect those buffers to be returned to
1947          * their normal dequeued state.
1948          */
1949         __vb2_queue_cancel(q);
1950         q->waiting_for_buffers = !q->is_output;
1951         q->last_buffer_dequeued = false;
1952
1953         dprintk(3, "successful\n");
1954         return 0;
1955 }
1956 EXPORT_SYMBOL_GPL(vb2_core_streamoff);
1957
1958 /**
1959  * __find_plane_by_offset() - find plane associated with the given offset off
1960  */
1961 static int __find_plane_by_offset(struct vb2_queue *q, unsigned long off,
1962                         unsigned int *_buffer, unsigned int *_plane)
1963 {
1964         struct vb2_buffer *vb;
1965         unsigned int buffer, plane;
1966
1967         /*
1968          * Go over all buffers and their planes, comparing the given offset
1969          * with an offset assigned to each plane. If a match is found,
1970          * return its buffer and plane numbers.
1971          */
1972         for (buffer = 0; buffer < q->num_buffers; ++buffer) {
1973                 vb = q->bufs[buffer];
1974
1975                 for (plane = 0; plane < vb->num_planes; ++plane) {
1976                         if (vb->planes[plane].m.offset == off) {
1977                                 *_buffer = buffer;
1978                                 *_plane = plane;
1979                                 return 0;
1980                         }
1981                 }
1982         }
1983
1984         return -EINVAL;
1985 }
1986
1987 /**
1988  * vb2_core_expbuf() - Export a buffer as a file descriptor
1989  * @q:          videobuf2 queue
1990  * @fd:         file descriptor associated with DMABUF (set by driver) *
1991  * @type:       buffer type
1992  * @index:      id number of the buffer
1993  * @plane:      index of the plane to be exported, 0 for single plane queues
1994  * @flags:      flags for newly created file, currently only O_CLOEXEC is
1995  *              supported, refer to manual of open syscall for more details
1996  *
1997  * The return values from this function are intended to be directly returned
1998  * from vidioc_expbuf handler in driver.
1999  */
2000 int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
2001                 unsigned int index, unsigned int plane, unsigned int flags)
2002 {
2003         struct vb2_buffer *vb = NULL;
2004         struct vb2_plane *vb_plane;
2005         int ret;
2006         struct dma_buf *dbuf;
2007
2008         if (q->memory != VB2_MEMORY_MMAP) {
2009                 dprintk(1, "queue is not currently set up for mmap\n");
2010                 return -EINVAL;
2011         }
2012
2013         if (!q->mem_ops->get_dmabuf) {
2014                 dprintk(1, "queue does not support DMA buffer exporting\n");
2015                 return -EINVAL;
2016         }
2017
2018         if (flags & ~(O_CLOEXEC | O_ACCMODE)) {
2019                 dprintk(1, "queue does support only O_CLOEXEC and access mode flags\n");
2020                 return -EINVAL;
2021         }
2022
2023         if (type != q->type) {
2024                 dprintk(1, "invalid buffer type\n");
2025                 return -EINVAL;
2026         }
2027
2028         if (index >= q->num_buffers) {
2029                 dprintk(1, "buffer index out of range\n");
2030                 return -EINVAL;
2031         }
2032
2033         vb = q->bufs[index];
2034
2035         if (plane >= vb->num_planes) {
2036                 dprintk(1, "buffer plane out of range\n");
2037                 return -EINVAL;
2038         }
2039
2040         if (vb2_fileio_is_active(q)) {
2041                 dprintk(1, "expbuf: file io in progress\n");
2042                 return -EBUSY;
2043         }
2044
2045         vb_plane = &vb->planes[plane];
2046
2047         dbuf = call_ptr_memop(vb, get_dmabuf, vb_plane->mem_priv,
2048                                 flags & O_ACCMODE);
2049         if (IS_ERR_OR_NULL(dbuf)) {
2050                 dprintk(1, "failed to export buffer %d, plane %d\n",
2051                         index, plane);
2052                 return -EINVAL;
2053         }
2054
2055         ret = dma_buf_fd(dbuf, flags & ~O_ACCMODE);
2056         if (ret < 0) {
2057                 dprintk(3, "buffer %d, plane %d failed to export (%d)\n",
2058                         index, plane, ret);
2059                 dma_buf_put(dbuf);
2060                 return ret;
2061         }
2062
2063         dprintk(3, "buffer %d, plane %d exported as %d descriptor\n",
2064                 index, plane, ret);
2065         *fd = ret;
2066
2067         return 0;
2068 }
2069 EXPORT_SYMBOL_GPL(vb2_core_expbuf);
2070
2071 /**
2072  * vb2_mmap() - map video buffers into application address space
2073  * @q:          videobuf2 queue
2074  * @vma:        vma passed to the mmap file operation handler in the driver
2075  *
2076  * Should be called from mmap file operation handler of a driver.
2077  * This function maps one plane of one of the available video buffers to
2078  * userspace. To map whole video memory allocated on reqbufs, this function
2079  * has to be called once per each plane per each buffer previously allocated.
2080  *
2081  * When the userspace application calls mmap, it passes to it an offset returned
2082  * to it earlier by the means of vidioc_querybuf handler. That offset acts as
2083  * a "cookie", which is then used to identify the plane to be mapped.
2084  * This function finds a plane with a matching offset and a mapping is performed
2085  * by the means of a provided memory operation.
2086  *
2087  * The return values from this function are intended to be directly returned
2088  * from the mmap handler in driver.
2089  */
2090 int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
2091 {
2092         unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
2093         struct vb2_buffer *vb;
2094         unsigned int buffer = 0, plane = 0;
2095         int ret;
2096         unsigned long length;
2097
2098         if (q->memory != VB2_MEMORY_MMAP) {
2099                 dprintk(1, "queue is not currently set up for mmap\n");
2100                 return -EINVAL;
2101         }
2102
2103         /*
2104          * Check memory area access mode.
2105          */
2106         if (!(vma->vm_flags & VM_SHARED)) {
2107                 dprintk(1, "invalid vma flags, VM_SHARED needed\n");
2108                 return -EINVAL;
2109         }
2110         if (q->is_output) {
2111                 if (!(vma->vm_flags & VM_WRITE)) {
2112                         dprintk(1, "invalid vma flags, VM_WRITE needed\n");
2113                         return -EINVAL;
2114                 }
2115         } else {
2116                 if (!(vma->vm_flags & VM_READ)) {
2117                         dprintk(1, "invalid vma flags, VM_READ needed\n");
2118                         return -EINVAL;
2119                 }
2120         }
2121         if (vb2_fileio_is_active(q)) {
2122                 dprintk(1, "mmap: file io in progress\n");
2123                 return -EBUSY;
2124         }
2125
2126         /*
2127          * Find the plane corresponding to the offset passed by userspace.
2128          */
2129         ret = __find_plane_by_offset(q, off, &buffer, &plane);
2130         if (ret)
2131                 return ret;
2132
2133         vb = q->bufs[buffer];
2134
2135         /*
2136          * MMAP requires page_aligned buffers.
2137          * The buffer length was page_aligned at __vb2_buf_mem_alloc(),
2138          * so, we need to do the same here.
2139          */
2140         length = PAGE_ALIGN(vb->planes[plane].length);
2141         if (length < (vma->vm_end - vma->vm_start)) {
2142                 dprintk(1,
2143                         "MMAP invalid, as it would overflow buffer length\n");
2144                 return -EINVAL;
2145         }
2146
2147         mutex_lock(&q->mmap_lock);
2148         ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma);
2149         mutex_unlock(&q->mmap_lock);
2150         if (ret)
2151                 return ret;
2152
2153         dprintk(3, "buffer %d, plane %d successfully mapped\n", buffer, plane);
2154         return 0;
2155 }
2156 EXPORT_SYMBOL_GPL(vb2_mmap);
2157
2158 #ifndef CONFIG_MMU
2159 unsigned long vb2_get_unmapped_area(struct vb2_queue *q,
2160                                     unsigned long addr,
2161                                     unsigned long len,
2162                                     unsigned long pgoff,
2163                                     unsigned long flags)
2164 {
2165         unsigned long off = pgoff << PAGE_SHIFT;
2166         struct vb2_buffer *vb;
2167         unsigned int buffer, plane;
2168         void *vaddr;
2169         int ret;
2170
2171         if (q->memory != VB2_MEMORY_MMAP) {
2172                 dprintk(1, "queue is not currently set up for mmap\n");
2173                 return -EINVAL;
2174         }
2175
2176         /*
2177          * Find the plane corresponding to the offset passed by userspace.
2178          */
2179         ret = __find_plane_by_offset(q, off, &buffer, &plane);
2180         if (ret)
2181                 return ret;
2182
2183         vb = q->bufs[buffer];
2184
2185         vaddr = vb2_plane_vaddr(vb, plane);
2186         return vaddr ? (unsigned long)vaddr : -EINVAL;
2187 }
2188 EXPORT_SYMBOL_GPL(vb2_get_unmapped_area);
2189 #endif
2190
2191 /**
2192  * vb2_core_queue_init() - initialize a videobuf2 queue
2193  * @q:          videobuf2 queue; this structure should be allocated in driver
2194  *
2195  * The vb2_queue structure should be allocated by the driver. The driver is
2196  * responsible of clearing it's content and setting initial values for some
2197  * required entries before calling this function.
2198  * q->ops, q->mem_ops, q->type and q->io_modes are mandatory. Please refer
2199  * to the struct vb2_queue description in include/media/videobuf2-core.h
2200  * for more information.
2201  */
2202 int vb2_core_queue_init(struct vb2_queue *q)
2203 {
2204         /*
2205          * Sanity check
2206          */
2207         if (WARN_ON(!q)                   ||
2208             WARN_ON(!q->ops)              ||
2209             WARN_ON(!q->mem_ops)          ||
2210             WARN_ON(!q->type)             ||
2211             WARN_ON(!q->io_modes)         ||
2212             WARN_ON(!q->ops->queue_setup) ||
2213             WARN_ON(!q->ops->buf_queue))
2214                 return -EINVAL;
2215
2216         INIT_LIST_HEAD(&q->queued_list);
2217         INIT_LIST_HEAD(&q->done_list);
2218         spin_lock_init(&q->done_lock);
2219         mutex_init(&q->mmap_lock);
2220         init_waitqueue_head(&q->done_wq);
2221
2222         if (q->buf_struct_size == 0)
2223                 q->buf_struct_size = sizeof(struct vb2_buffer);
2224
2225         return 0;
2226 }
2227 EXPORT_SYMBOL_GPL(vb2_core_queue_init);
2228
2229 static int __vb2_init_fileio(struct vb2_queue *q, int read);
2230 static int __vb2_cleanup_fileio(struct vb2_queue *q);
2231 /**
2232  * vb2_core_queue_release() - stop streaming, release the queue and free memory
2233  * @q:          videobuf2 queue
2234  *
2235  * This function stops streaming and performs necessary clean ups, including
2236  * freeing video buffer memory. The driver is responsible for freeing
2237  * the vb2_queue structure itself.
2238  */
2239 void vb2_core_queue_release(struct vb2_queue *q)
2240 {
2241         __vb2_cleanup_fileio(q);
2242         __vb2_queue_cancel(q);
2243         mutex_lock(&q->mmap_lock);
2244         __vb2_queue_free(q, q->num_buffers);
2245         mutex_unlock(&q->mmap_lock);
2246 }
2247 EXPORT_SYMBOL_GPL(vb2_core_queue_release);
2248
2249 /**
2250  * vb2_core_poll() - implements poll userspace operation
2251  * @q:          videobuf2 queue
2252  * @file:       file argument passed to the poll file operation handler
2253  * @wait:       wait argument passed to the poll file operation handler
2254  *
2255  * This function implements poll file operation handler for a driver.
2256  * For CAPTURE queues, if a buffer is ready to be dequeued, the userspace will
2257  * be informed that the file descriptor of a video device is available for
2258  * reading.
2259  * For OUTPUT queues, if a buffer is ready to be dequeued, the file descriptor
2260  * will be reported as available for writing.
2261  *
2262  * The return values from this function are intended to be directly returned
2263  * from poll handler in driver.
2264  */
2265 unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
2266                 poll_table *wait)
2267 {
2268         unsigned long req_events = poll_requested_events(wait);
2269         struct vb2_buffer *vb = NULL;
2270         unsigned long flags;
2271
2272         if (!q->is_output && !(req_events & (POLLIN | POLLRDNORM)))
2273                 return 0;
2274         if (q->is_output && !(req_events & (POLLOUT | POLLWRNORM)))
2275                 return 0;
2276
2277         /*
2278          * Start file I/O emulator only if streaming API has not been used yet.
2279          */
2280         if (q->num_buffers == 0 && !vb2_fileio_is_active(q)) {
2281                 if (!q->is_output && (q->io_modes & VB2_READ) &&
2282                                 (req_events & (POLLIN | POLLRDNORM))) {
2283                         if (__vb2_init_fileio(q, 1))
2284                                 return POLLERR;
2285                 }
2286                 if (q->is_output && (q->io_modes & VB2_WRITE) &&
2287                                 (req_events & (POLLOUT | POLLWRNORM))) {
2288                         if (__vb2_init_fileio(q, 0))
2289                                 return POLLERR;
2290                         /*
2291                          * Write to OUTPUT queue can be done immediately.
2292                          */
2293                         return POLLOUT | POLLWRNORM;
2294                 }
2295         }
2296
2297         /*
2298          * There is nothing to wait for if the queue isn't streaming, or if the
2299          * error flag is set.
2300          */
2301         if (!vb2_is_streaming(q) || q->error)
2302                 return POLLERR;
2303
2304         /*
2305          * If this quirk is set and QBUF hasn't been called yet then
2306          * return POLLERR as well. This only affects capture queues, output
2307          * queues will always initialize waiting_for_buffers to false.
2308          * This quirk is set by V4L2 for backwards compatibility reasons.
2309          */
2310         if (q->quirk_poll_must_check_waiting_for_buffers &&
2311             q->waiting_for_buffers && (req_events & (POLLIN | POLLRDNORM)))
2312                 return POLLERR;
2313
2314         /*
2315          * For output streams you can call write() as long as there are fewer
2316          * buffers queued than there are buffers available.
2317          */
2318         if (q->is_output && q->fileio && q->queued_count < q->num_buffers)
2319                 return POLLOUT | POLLWRNORM;
2320
2321         if (list_empty(&q->done_list)) {
2322                 /*
2323                  * If the last buffer was dequeued from a capture queue,
2324                  * return immediately. DQBUF will return -EPIPE.
2325                  */
2326                 if (q->last_buffer_dequeued)
2327                         return POLLIN | POLLRDNORM;
2328
2329                 poll_wait(file, &q->done_wq, wait);
2330         }
2331
2332         /*
2333          * Take first buffer available for dequeuing.
2334          */
2335         spin_lock_irqsave(&q->done_lock, flags);
2336         if (!list_empty(&q->done_list))
2337                 vb = list_first_entry(&q->done_list, struct vb2_buffer,
2338                                         done_entry);
2339         spin_unlock_irqrestore(&q->done_lock, flags);
2340
2341         if (vb && (vb->state == VB2_BUF_STATE_DONE
2342                         || vb->state == VB2_BUF_STATE_ERROR)) {
2343                 return (q->is_output) ?
2344                                 POLLOUT | POLLWRNORM :
2345                                 POLLIN | POLLRDNORM;
2346         }
2347         return 0;
2348 }
2349 EXPORT_SYMBOL_GPL(vb2_core_poll);
2350
2351 /**
2352  * struct vb2_fileio_buf - buffer context used by file io emulator
2353  *
2354  * vb2 provides a compatibility layer and emulator of file io (read and
2355  * write) calls on top of streaming API. This structure is used for
2356  * tracking context related to the buffers.
2357  */
2358 struct vb2_fileio_buf {
2359         void *vaddr;
2360         unsigned int size;
2361         unsigned int pos;
2362         unsigned int queued:1;
2363 };
2364
2365 /**
2366  * struct vb2_fileio_data - queue context used by file io emulator
2367  *
2368  * @cur_index:  the index of the buffer currently being read from or
2369  *              written to. If equal to q->num_buffers then a new buffer
2370  *              must be dequeued.
2371  * @initial_index: in the read() case all buffers are queued up immediately
2372  *              in __vb2_init_fileio() and __vb2_perform_fileio() just cycles
2373  *              buffers. However, in the write() case no buffers are initially
2374  *              queued, instead whenever a buffer is full it is queued up by
2375  *              __vb2_perform_fileio(). Only once all available buffers have
2376  *              been queued up will __vb2_perform_fileio() start to dequeue
2377  *              buffers. This means that initially __vb2_perform_fileio()
2378  *              needs to know what buffer index to use when it is queuing up
2379  *              the buffers for the first time. That initial index is stored
2380  *              in this field. Once it is equal to q->num_buffers all
2381  *              available buffers have been queued and __vb2_perform_fileio()
2382  *              should start the normal dequeue/queue cycle.
2383  *
2384  * vb2 provides a compatibility layer and emulator of file io (read and
2385  * write) calls on top of streaming API. For proper operation it required
2386  * this structure to save the driver state between each call of the read
2387  * or write function.
2388  */
2389 struct vb2_fileio_data {
2390         unsigned int count;
2391         unsigned int type;
2392         unsigned int memory;
2393         struct vb2_fileio_buf bufs[VB2_MAX_FRAME];
2394         unsigned int cur_index;
2395         unsigned int initial_index;
2396         unsigned int q_count;
2397         unsigned int dq_count;
2398         unsigned read_once:1;
2399         unsigned write_immediately:1;
2400 };
2401
2402 /**
2403  * __vb2_init_fileio() - initialize file io emulator
2404  * @q:          videobuf2 queue
2405  * @read:       mode selector (1 means read, 0 means write)
2406  */
2407 static int __vb2_init_fileio(struct vb2_queue *q, int read)
2408 {
2409         struct vb2_fileio_data *fileio;
2410         int i, ret;
2411         unsigned int count = 0;
2412
2413         /*
2414          * Sanity check
2415          */
2416         if (WARN_ON((read && !(q->io_modes & VB2_READ)) ||
2417                     (!read && !(q->io_modes & VB2_WRITE))))
2418                 return -EINVAL;
2419
2420         /*
2421          * Check if device supports mapping buffers to kernel virtual space.
2422          */
2423         if (!q->mem_ops->vaddr)
2424                 return -EBUSY;
2425
2426         /*
2427          * Check if streaming api has not been already activated.
2428          */
2429         if (q->streaming || q->num_buffers > 0)
2430                 return -EBUSY;
2431
2432         /*
2433          * Start with count 1, driver can increase it in queue_setup()
2434          */
2435         count = 1;
2436
2437         dprintk(3, "setting up file io: mode %s, count %d, read_once %d, write_immediately %d\n",
2438                 (read) ? "read" : "write", count, q->fileio_read_once,
2439                 q->fileio_write_immediately);
2440
2441         fileio = kzalloc(sizeof(*fileio), GFP_KERNEL);
2442         if (fileio == NULL)
2443                 return -ENOMEM;
2444
2445         fileio->read_once = q->fileio_read_once;
2446         fileio->write_immediately = q->fileio_write_immediately;
2447
2448         /*
2449          * Request buffers and use MMAP type to force driver
2450          * to allocate buffers by itself.
2451          */
2452         fileio->count = count;
2453         fileio->memory = VB2_MEMORY_MMAP;
2454         fileio->type = q->type;
2455         q->fileio = fileio;
2456         ret = vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2457         if (ret)
2458                 goto err_kfree;
2459
2460         /*
2461          * Check if plane_count is correct
2462          * (multiplane buffers are not supported).
2463          */
2464         if (q->bufs[0]->num_planes != 1) {
2465                 ret = -EBUSY;
2466                 goto err_reqbufs;
2467         }
2468
2469         /*
2470          * Get kernel address of each buffer.
2471          */
2472         for (i = 0; i < q->num_buffers; i++) {
2473                 fileio->bufs[i].vaddr = vb2_plane_vaddr(q->bufs[i], 0);
2474                 if (fileio->bufs[i].vaddr == NULL) {
2475                         ret = -EINVAL;
2476                         goto err_reqbufs;
2477                 }
2478                 fileio->bufs[i].size = vb2_plane_size(q->bufs[i], 0);
2479         }
2480
2481         /*
2482          * Read mode requires pre queuing of all buffers.
2483          */
2484         if (read) {
2485                 /*
2486                  * Queue all buffers.
2487                  */
2488                 for (i = 0; i < q->num_buffers; i++) {
2489                         ret = vb2_core_qbuf(q, i, NULL);
2490                         if (ret)
2491                                 goto err_reqbufs;
2492                         fileio->bufs[i].queued = 1;
2493                 }
2494                 /*
2495                  * All buffers have been queued, so mark that by setting
2496                  * initial_index to q->num_buffers
2497                  */
2498                 fileio->initial_index = q->num_buffers;
2499                 fileio->cur_index = q->num_buffers;
2500         }
2501
2502         /*
2503          * Start streaming.
2504          */
2505         ret = vb2_core_streamon(q, q->type);
2506         if (ret)
2507                 goto err_reqbufs;
2508
2509         return ret;
2510
2511 err_reqbufs:
2512         fileio->count = 0;
2513         vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2514
2515 err_kfree:
2516         q->fileio = NULL;
2517         kfree(fileio);
2518         return ret;
2519 }
2520
2521 /**
2522  * __vb2_cleanup_fileio() - free resourced used by file io emulator
2523  * @q:          videobuf2 queue
2524  */
2525 static int __vb2_cleanup_fileio(struct vb2_queue *q)
2526 {
2527         struct vb2_fileio_data *fileio = q->fileio;
2528
2529         if (fileio) {
2530                 vb2_core_streamoff(q, q->type);
2531                 q->fileio = NULL;
2532                 fileio->count = 0;
2533                 vb2_core_reqbufs(q, fileio->memory, &fileio->count);
2534                 kfree(fileio);
2535                 dprintk(3, "file io emulator closed\n");
2536         }
2537         return 0;
2538 }
2539
2540 /**
2541  * __vb2_perform_fileio() - perform a single file io (read or write) operation
2542  * @q:          videobuf2 queue
2543  * @data:       pointed to target userspace buffer
2544  * @count:      number of bytes to read or write
2545  * @ppos:       file handle position tracking pointer
2546  * @nonblock:   mode selector (1 means blocking calls, 0 means nonblocking)
2547  * @read:       access mode selector (1 means read, 0 means write)
2548  */
2549 static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
2550                 loff_t *ppos, int nonblock, int read)
2551 {
2552         struct vb2_fileio_data *fileio;
2553         struct vb2_fileio_buf *buf;
2554         bool is_multiplanar = q->is_multiplanar;
2555         /*
2556          * When using write() to write data to an output video node the vb2 core
2557          * should copy timestamps if V4L2_BUF_FLAG_TIMESTAMP_COPY is set. Nobody
2558          * else is able to provide this information with the write() operation.
2559          */
2560         bool copy_timestamp = !read && q->copy_timestamp;
2561         unsigned index;
2562         int ret;
2563
2564         dprintk(3, "mode %s, offset %ld, count %zd, %sblocking\n",
2565                 read ? "read" : "write", (long)*ppos, count,
2566                 nonblock ? "non" : "");
2567
2568         if (!data)
2569                 return -EINVAL;
2570
2571         /*
2572          * Initialize emulator on first call.
2573          */
2574         if (!vb2_fileio_is_active(q)) {
2575                 ret = __vb2_init_fileio(q, read);
2576                 dprintk(3, "vb2_init_fileio result: %d\n", ret);
2577                 if (ret)
2578                         return ret;
2579         }
2580         fileio = q->fileio;
2581
2582         /*
2583          * Check if we need to dequeue the buffer.
2584          */
2585         index = fileio->cur_index;
2586         if (index >= q->num_buffers) {
2587                 struct vb2_buffer *b;
2588
2589                 /*
2590                  * Call vb2_dqbuf to get buffer back.
2591                  */
2592                 ret = vb2_core_dqbuf(q, &index, NULL, nonblock);
2593                 dprintk(5, "vb2_dqbuf result: %d\n", ret);
2594                 if (ret)
2595                         return ret;
2596                 fileio->dq_count += 1;
2597
2598                 fileio->cur_index = index;
2599                 buf = &fileio->bufs[index];
2600                 b = q->bufs[index];
2601
2602                 /*
2603                  * Get number of bytes filled by the driver
2604                  */
2605                 buf->pos = 0;
2606                 buf->queued = 0;
2607                 buf->size = read ? vb2_get_plane_payload(q->bufs[index], 0)
2608                                  : vb2_plane_size(q->bufs[index], 0);
2609                 /* Compensate for data_offset on read in the multiplanar case. */
2610                 if (is_multiplanar && read &&
2611                                 b->planes[0].data_offset < buf->size) {
2612                         buf->pos = b->planes[0].data_offset;
2613                         buf->size -= buf->pos;
2614                 }
2615         } else {
2616                 buf = &fileio->bufs[index];
2617         }
2618
2619         /*
2620          * Limit count on last few bytes of the buffer.
2621          */
2622         if (buf->pos + count > buf->size) {
2623                 count = buf->size - buf->pos;
2624                 dprintk(5, "reducing read count: %zd\n", count);
2625         }
2626
2627         /*
2628          * Transfer data to userspace.
2629          */
2630         dprintk(3, "copying %zd bytes - buffer %d, offset %u\n",
2631                 count, index, buf->pos);
2632         if (read)
2633                 ret = copy_to_user(data, buf->vaddr + buf->pos, count);
2634         else
2635                 ret = copy_from_user(buf->vaddr + buf->pos, data, count);
2636         if (ret) {
2637                 dprintk(3, "error copying data\n");
2638                 return -EFAULT;
2639         }
2640
2641         /*
2642          * Update counters.
2643          */
2644         buf->pos += count;
2645         *ppos += count;
2646
2647         /*
2648          * Queue next buffer if required.
2649          */
2650         if (buf->pos == buf->size || (!read && fileio->write_immediately)) {
2651                 struct vb2_buffer *b = q->bufs[index];
2652
2653                 /*
2654                  * Check if this is the last buffer to read.
2655                  */
2656                 if (read && fileio->read_once && fileio->dq_count == 1) {
2657                         dprintk(3, "read limit reached\n");
2658                         return __vb2_cleanup_fileio(q);
2659                 }
2660
2661                 /*
2662                  * Call vb2_qbuf and give buffer to the driver.
2663                  */
2664                 b->planes[0].bytesused = buf->pos;
2665
2666                 if (copy_timestamp)
2667                         b->timestamp = ktime_get_ns();
2668                 ret = vb2_core_qbuf(q, index, NULL);
2669                 dprintk(5, "vb2_dbuf result: %d\n", ret);
2670                 if (ret)
2671                         return ret;
2672
2673                 /*
2674                  * Buffer has been queued, update the status
2675                  */
2676                 buf->pos = 0;
2677                 buf->queued = 1;
2678                 buf->size = vb2_plane_size(q->bufs[index], 0);
2679                 fileio->q_count += 1;
2680                 /*
2681                  * If we are queuing up buffers for the first time, then
2682                  * increase initial_index by one.
2683                  */
2684                 if (fileio->initial_index < q->num_buffers)
2685                         fileio->initial_index++;
2686                 /*
2687                  * The next buffer to use is either a buffer that's going to be
2688                  * queued for the first time (initial_index < q->num_buffers)
2689                  * or it is equal to q->num_buffers, meaning that the next
2690                  * time we need to dequeue a buffer since we've now queued up
2691                  * all the 'first time' buffers.
2692                  */
2693                 fileio->cur_index = fileio->initial_index;
2694         }
2695
2696         /*
2697          * Return proper number of bytes processed.
2698          */
2699         if (ret == 0)
2700                 ret = count;
2701         return ret;
2702 }
2703
2704 size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
2705                 loff_t *ppos, int nonblocking)
2706 {
2707         return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1);
2708 }
2709 EXPORT_SYMBOL_GPL(vb2_read);
2710
2711 size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
2712                 loff_t *ppos, int nonblocking)
2713 {
2714         return __vb2_perform_fileio(q, (char __user *) data, count,
2715                                                         ppos, nonblocking, 0);
2716 }
2717 EXPORT_SYMBOL_GPL(vb2_write);
2718
2719 struct vb2_threadio_data {
2720         struct task_struct *thread;
2721         vb2_thread_fnc fnc;
2722         void *priv;
2723         bool stop;
2724 };
2725
2726 static int vb2_thread(void *data)
2727 {
2728         struct vb2_queue *q = data;
2729         struct vb2_threadio_data *threadio = q->threadio;
2730         bool copy_timestamp = false;
2731         unsigned prequeue = 0;
2732         unsigned index = 0;
2733         int ret = 0;
2734
2735         if (q->is_output) {
2736                 prequeue = q->num_buffers;
2737                 copy_timestamp = q->copy_timestamp;
2738         }
2739
2740         set_freezable();
2741
2742         for (;;) {
2743                 struct vb2_buffer *vb;
2744
2745                 /*
2746                  * Call vb2_dqbuf to get buffer back.
2747                  */
2748                 if (prequeue) {
2749                         vb = q->bufs[index++];
2750                         prequeue--;
2751                 } else {
2752                         call_void_qop(q, wait_finish, q);
2753                         if (!threadio->stop)
2754                                 ret = vb2_core_dqbuf(q, &index, NULL, 0);
2755                         call_void_qop(q, wait_prepare, q);
2756                         dprintk(5, "file io: vb2_dqbuf result: %d\n", ret);
2757                         if (!ret)
2758                                 vb = q->bufs[index];
2759                 }
2760                 if (ret || threadio->stop)
2761                         break;
2762                 try_to_freeze();
2763
2764                 if (vb->state != VB2_BUF_STATE_ERROR)
2765                         if (threadio->fnc(vb, threadio->priv))
2766                                 break;
2767                 call_void_qop(q, wait_finish, q);
2768                 if (copy_timestamp)
2769                         vb->timestamp = ktime_get_ns();;
2770                 if (!threadio->stop)
2771                         ret = vb2_core_qbuf(q, vb->index, NULL);
2772                 call_void_qop(q, wait_prepare, q);
2773                 if (ret || threadio->stop)
2774                         break;
2775         }
2776
2777         /* Hmm, linux becomes *very* unhappy without this ... */
2778         while (!kthread_should_stop()) {
2779                 set_current_state(TASK_INTERRUPTIBLE);
2780                 schedule();
2781         }
2782         return 0;
2783 }
2784
2785 /*
2786  * This function should not be used for anything else but the videobuf2-dvb
2787  * support. If you think you have another good use-case for this, then please
2788  * contact the linux-media mailinglist first.
2789  */
2790 int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
2791                      const char *thread_name)
2792 {
2793         struct vb2_threadio_data *threadio;
2794         int ret = 0;
2795
2796         if (q->threadio)
2797                 return -EBUSY;
2798         if (vb2_is_busy(q))
2799                 return -EBUSY;
2800         if (WARN_ON(q->fileio))
2801                 return -EBUSY;
2802
2803         threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
2804         if (threadio == NULL)
2805                 return -ENOMEM;
2806         threadio->fnc = fnc;
2807         threadio->priv = priv;
2808
2809         ret = __vb2_init_fileio(q, !q->is_output);
2810         dprintk(3, "file io: vb2_init_fileio result: %d\n", ret);
2811         if (ret)
2812                 goto nomem;
2813         q->threadio = threadio;
2814         threadio->thread = kthread_run(vb2_thread, q, "vb2-%s", thread_name);
2815         if (IS_ERR(threadio->thread)) {
2816                 ret = PTR_ERR(threadio->thread);
2817                 threadio->thread = NULL;
2818                 goto nothread;
2819         }
2820         return 0;
2821
2822 nothread:
2823         __vb2_cleanup_fileio(q);
2824 nomem:
2825         kfree(threadio);
2826         return ret;
2827 }
2828 EXPORT_SYMBOL_GPL(vb2_thread_start);
2829
2830 int vb2_thread_stop(struct vb2_queue *q)
2831 {
2832         struct vb2_threadio_data *threadio = q->threadio;
2833         int err;
2834
2835         if (threadio == NULL)
2836                 return 0;
2837         threadio->stop = true;
2838         /* Wake up all pending sleeps in the thread */
2839         vb2_queue_error(q);
2840         err = kthread_stop(threadio->thread);
2841         __vb2_cleanup_fileio(q);
2842         threadio->thread = NULL;
2843         kfree(threadio);
2844         q->threadio = NULL;
2845         return err;
2846 }
2847 EXPORT_SYMBOL_GPL(vb2_thread_stop);
2848
2849 MODULE_DESCRIPTION("Media buffer core framework");
2850 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>, Marek Szyprowski");
2851 MODULE_LICENSE("GPL");