drivers/staging/android/ion/ion.c

   1 /*
   2
   3  * drivers/staging/android/ion/ion.c
   4  *
   5  * Copyright (C) 2011 Google, Inc.
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #include <linux/device.h>
  19 #include <linux/err.h>
  20 #include <linux/file.h>
  21 #include <linux/freezer.h>
  22 #include <linux/fs.h>
  23 #include <linux/anon_inodes.h>
  24 #include <linux/kthread.h>
  25 #include <linux/list.h>
  26 #include <linux/memblock.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/export.h>
  29 #include <linux/mm.h>
  30 #include <linux/mm_types.h>
  31 #include <linux/rbtree.h>
  32 #include <linux/slab.h>
  33 #include <linux/seq_file.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/debugfs.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/idr.h>
  39
  40 #include "ion.h"
  41 #include "ion_priv.h"
  42 #include "compat_ion.h"
  43
  44 /**
  45  * struct ion_device - the metadata of the ion device node
  46  * @dev:                the actual misc device
  47  * @buffers:            an rb tree of all the existing buffers
  48  * @buffer_lock:        lock protecting the tree of buffers
  49  * @lock:               rwsem protecting the tree of heaps and clients
  50  * @heaps:              list of all the heaps in the system
  51  * @user_clients:       list of all the clients created from userspace
  52  */
  53 struct ion_device {
  54         struct miscdevice dev;
  55         struct rb_root buffers;
  56         struct mutex buffer_lock;
  57         struct rw_semaphore lock;
  58         struct plist_head heaps;
  59         long (*custom_ioctl)(struct ion_client *client, unsigned int cmd,
  60                              unsigned long arg);
  61         struct rb_root clients;
  62         struct dentry *debug_root;
  63         struct dentry *heaps_debug_root;
  64         struct dentry *clients_debug_root;
  65 };
  66
  67 /**
  68  * struct ion_client - a process/hw block local address space
  69  * @node:               node in the tree of all clients
  70  * @dev:                backpointer to ion device
  71  * @handles:            an rb tree of all the handles in this client
  72  * @idr:                an idr space for allocating handle ids
  73  * @lock:               lock protecting the tree of handles
  74  * @name:               used for debugging
  75  * @task:               used for debugging
  76  *
  77  * A client represents a list of buffers this client may access.
  78  * The mutex stored here is used to protect both handles tree
  79  * as well as the handles themselves, and should be held while modifying either.
  80  */
  81 struct ion_client {
  82         struct rb_node node;
  83         struct ion_device *dev;
  84         struct rb_root handles;
  85         struct idr idr;
  86         struct mutex lock;
  87         const char *name;
  88         struct task_struct *task;
  89         pid_t pid;
  90         struct dentry *debug_root;
  91 };
  92
  93 /**
  94  * ion_handle - a client local reference to a buffer
  95  * @ref:                reference count
  96  * @client:             back pointer to the client the buffer resides in
  97  * @buffer:             pointer to the buffer
  98  * @node:               node in the client's handle rbtree
  99  * @kmap_cnt:           count of times this client has mapped to kernel
 100  * @id:                 client-unique id allocated by client->idr
 101  *
 102  * Modifications to node, map_cnt or mapping should be protected by the
 103  * lock in the client.  Other fields are never changed after initialization.
 104  */
 105 struct ion_handle {
 106         struct kref ref;
 107         struct ion_client *client;
 108         struct ion_buffer *buffer;
 109         struct rb_node node;
 110         unsigned int kmap_cnt;
 111         int id;
 112 };
 113
 114 bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer)
 115 {
 116         return (buffer->flags & ION_FLAG_CACHED) &&
 117                 !(buffer->flags & ION_FLAG_CACHED_NEEDS_SYNC);
 118 }
 119
 120 bool ion_buffer_cached(struct ion_buffer *buffer)
 121 {
 122         return !!(buffer->flags & ION_FLAG_CACHED);
 123 }
 124
 125 static inline struct page *ion_buffer_page(struct page *page)
 126 {
 127         return (struct page *)((unsigned long)page & ~(1UL));
 128 }
 129
 130 static inline bool ion_buffer_page_is_dirty(struct page *page)
 131 {
 132         return !!((unsigned long)page & 1UL);
 133 }
 134
 135 static inline void ion_buffer_page_dirty(struct page **page)
 136 {
 137         *page = (struct page *)((unsigned long)(*page) | 1UL);
 138 }
 139
 140 static inline void ion_buffer_page_clean(struct page **page)
 141 {
 142         *page = (struct page *)((unsigned long)(*page) & ~(1UL));
 143 }
 144
 145 /* this function should only be called while dev->lock is held */
 146 static void ion_buffer_add(struct ion_device *dev,
 147                            struct ion_buffer *buffer)
 148 {
 149         struct rb_node **p = &dev->buffers.rb_node;
 150         struct rb_node *parent = NULL;
 151         struct ion_buffer *entry;
 152
 153         while (*p) {
 154                 parent = *p;
 155                 entry = rb_entry(parent, struct ion_buffer, node);
 156
 157                 if (buffer < entry) {
 158                         p = &(*p)->rb_left;
 159                 } else if (buffer > entry) {
 160                         p = &(*p)->rb_right;
 161                 } else {
 162                         pr_err("%s: buffer already found.", __func__);
 163                         BUG();
 164                 }
 165         }
 166
 167         rb_link_node(&buffer->node, parent, p);
 168         rb_insert_color(&buffer->node, &dev->buffers);
 169 }
 170
 171 /* this function should only be called while dev->lock is held */
 172 static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 173                                      struct ion_device *dev,
 174                                      unsigned long len,
 175                                      unsigned long align,
 176                                      unsigned long flags)
 177 {
 178         struct ion_buffer *buffer;
 179         struct sg_table *table;
 180         struct scatterlist *sg;
 181         int i, ret;
 182
 183         buffer = kzalloc(sizeof(struct ion_buffer), GFP_KERNEL);
 184         if (!buffer)
 185                 return ERR_PTR(-ENOMEM);
 186
 187         buffer->heap = heap;
 188         buffer->flags = flags;
 189         kref_init(&buffer->ref);
 190
 191         ret = heap->ops->allocate(heap, buffer, len, align, flags);
 192
 193         if (ret) {
 194                 if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))
 195                         goto err2;
 196
 197                 ion_heap_freelist_drain(heap, 0);
 198                 ret = heap->ops->allocate(heap, buffer, len, align,
 199                                           flags);
 200                 if (ret)
 201                         goto err2;
 202         }
 203
 204         buffer->dev = dev;
 205         buffer->size = len;
 206
 207         table = heap->ops->map_dma(heap, buffer);
 208         if (WARN_ONCE(table == NULL,
 209                         "heap->ops->map_dma should return ERR_PTR on error"))
 210                 table = ERR_PTR(-EINVAL);
 211         if (IS_ERR(table)) {
 212                 heap->ops->free(buffer);
 213                 kfree(buffer);
 214                 return ERR_PTR(PTR_ERR(table));
 215         }
 216         buffer->sg_table = table;
 217         if (ion_buffer_fault_user_mappings(buffer)) {
 218                 int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
 219                 struct scatterlist *sg;
 220                 int i, j, k = 0;
 221
 222                 buffer->pages = vmalloc(sizeof(struct page *) * num_pages);
 223                 if (!buffer->pages) {
 224                         ret = -ENOMEM;
 225                         goto err1;
 226                 }
 227
 228                 for_each_sg(table->sgl, sg, table->nents, i) {
 229                         struct page *page = sg_page(sg);
 230
 231                         for (j = 0; j < sg->length / PAGE_SIZE; j++)
 232                                 buffer->pages[k++] = page++;
 233                 }
 234
 235                 if (ret)
 236                         goto err;
 237         }
 238
 239         buffer->dev = dev;
 240         buffer->size = len;
 241         INIT_LIST_HEAD(&buffer->vmas);
 242         mutex_init(&buffer->lock);
 243         /* this will set up dma addresses for the sglist -- it is not
 244            technically correct as per the dma api -- a specific
 245            device isn't really taking ownership here.  However, in practice on
 246            our systems the only dma_address space is physical addresses.
 247            Additionally, we can't afford the overhead of invalidating every
 248            allocation via dma_map_sg. The implicit contract here is that
 249            memory comming from the heaps is ready for dma, ie if it has a
 250            cached mapping that mapping has been invalidated */
 251         for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i)
 252                 sg_dma_address(sg) = sg_phys(sg);
 253         mutex_lock(&dev->buffer_lock);
 254         ion_buffer_add(dev, buffer);
 255         mutex_unlock(&dev->buffer_lock);
 256         return buffer;
 257
 258 err:
 259         heap->ops->unmap_dma(heap, buffer);
 260         heap->ops->free(buffer);
 261 err1:
 262         if (buffer->pages)
 263                 vfree(buffer->pages);
 264 err2:
 265         kfree(buffer);
 266         return ERR_PTR(ret);
 267 }
 268
 269 void ion_buffer_destroy(struct ion_buffer *buffer)
 270 {
 271         if (WARN_ON(buffer->kmap_cnt > 0))
 272                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 273         buffer->heap->ops->unmap_dma(buffer->heap, buffer);
 274         buffer->heap->ops->free(buffer);
 275         if (buffer->pages)
 276                 vfree(buffer->pages);
 277         kfree(buffer);
 278 }
 279
 280 static void _ion_buffer_destroy(struct kref *kref)
 281 {
 282         struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
 283         struct ion_heap *heap = buffer->heap;
 284         struct ion_device *dev = buffer->dev;
 285
 286         mutex_lock(&dev->buffer_lock);
 287         rb_erase(&buffer->node, &dev->buffers);
 288         mutex_unlock(&dev->buffer_lock);
 289
 290         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 291                 ion_heap_freelist_add(heap, buffer);
 292         else
 293                 ion_buffer_destroy(buffer);
 294 }
 295
 296 static void ion_buffer_get(struct ion_buffer *buffer)
 297 {
 298         kref_get(&buffer->ref);
 299 }
 300
 301 static int ion_buffer_put(struct ion_buffer *buffer)
 302 {
 303         return kref_put(&buffer->ref, _ion_buffer_destroy);
 304 }
 305
 306 static void ion_buffer_add_to_handle(struct ion_buffer *buffer)
 307 {
 308         mutex_lock(&buffer->lock);
 309         buffer->handle_count++;
 310         mutex_unlock(&buffer->lock);
 311 }
 312
 313 static void ion_buffer_remove_from_handle(struct ion_buffer *buffer)
 314 {
 315         /*
 316          * when a buffer is removed from a handle, if it is not in
 317          * any other handles, copy the taskcomm and the pid of the
 318          * process it's being removed from into the buffer.  At this
 319          * point there will be no way to track what processes this buffer is
 320          * being used by, it only exists as a dma_buf file descriptor.
 321          * The taskcomm and pid can provide a debug hint as to where this fd
 322          * is in the system
 323          */
 324         mutex_lock(&buffer->lock);
 325         buffer->handle_count--;
 326         BUG_ON(buffer->handle_count < 0);
 327         if (!buffer->handle_count) {
 328                 struct task_struct *task;
 329
 330                 task = current->group_leader;
 331                 get_task_comm(buffer->task_comm, task);
 332                 buffer->pid = task_pid_nr(task);
 333         }
 334         mutex_unlock(&buffer->lock);
 335 }
 336
 337 static struct ion_handle *ion_handle_create(struct ion_client *client,
 338                                      struct ion_buffer *buffer)
 339 {
 340         struct ion_handle *handle;
 341
 342         handle = kzalloc(sizeof(struct ion_handle), GFP_KERNEL);
 343         if (!handle)
 344                 return ERR_PTR(-ENOMEM);
 345         kref_init(&handle->ref);
 346         RB_CLEAR_NODE(&handle->node);
 347         handle->client = client;
 348         ion_buffer_get(buffer);
 349         ion_buffer_add_to_handle(buffer);
 350         handle->buffer = buffer;
 351
 352         return handle;
 353 }
 354
 355 static void ion_handle_kmap_put(struct ion_handle *);
 356
 357 static void ion_handle_destroy(struct kref *kref)
 358 {
 359         struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
 360         struct ion_client *client = handle->client;
 361         struct ion_buffer *buffer = handle->buffer;
 362
 363         mutex_lock(&buffer->lock);
 364         while (handle->kmap_cnt)
 365                 ion_handle_kmap_put(handle);
 366         mutex_unlock(&buffer->lock);
 367
 368         idr_remove(&client->idr, handle->id);
 369         if (!RB_EMPTY_NODE(&handle->node))
 370                 rb_erase(&handle->node, &client->handles);
 371
 372         ion_buffer_remove_from_handle(buffer);
 373         ion_buffer_put(buffer);
 374
 375         kfree(handle);
 376 }
 377
 378 struct ion_buffer *ion_handle_buffer(struct ion_handle *handle)
 379 {
 380         return handle->buffer;
 381 }
 382
 383 static void ion_handle_get(struct ion_handle *handle)
 384 {
 385         kref_get(&handle->ref);
 386 }
 387
 388 static int ion_handle_put(struct ion_handle *handle)
 389 {
 390         struct ion_client *client = handle->client;
 391         int ret;
 392
 393         mutex_lock(&client->lock);
 394         ret = kref_put(&handle->ref, ion_handle_destroy);
 395         mutex_unlock(&client->lock);
 396
 397         return ret;
 398 }
 399
 400 static struct ion_handle *ion_handle_lookup(struct ion_client *client,
 401                                             struct ion_buffer *buffer)
 402 {
 403         struct rb_node *n = client->handles.rb_node;
 404
 405         while (n) {
 406                 struct ion_handle *entry = rb_entry(n, struct ion_handle, node);
 407                 if (buffer < entry->buffer)
 408                         n = n->rb_left;
 409                 else if (buffer > entry->buffer)
 410                         n = n->rb_right;
 411                 else
 412                         return entry;
 413         }
 414         return ERR_PTR(-EINVAL);
 415 }
 416
 417 static struct ion_handle *ion_handle_get_by_id(struct ion_client *client,
 418                                                 int id)
 419 {
 420         struct ion_handle *handle;
 421
 422         mutex_lock(&client->lock);
 423         handle = idr_find(&client->idr, id);
 424         if (handle)
 425                 ion_handle_get(handle);
 426         mutex_unlock(&client->lock);
 427
 428         return handle ? handle : ERR_PTR(-EINVAL);
 429 }
 430
 431 static bool ion_handle_validate(struct ion_client *client,
 432                                 struct ion_handle *handle)
 433 {
 434         WARN_ON(!mutex_is_locked(&client->lock));
 435         return idr_find(&client->idr, handle->id) == handle;
 436 }
 437
 438 static int ion_handle_add(struct ion_client *client, struct ion_handle *handle)
 439 {
 440         int id;
 441         struct rb_node **p = &client->handles.rb_node;
 442         struct rb_node *parent = NULL;
 443         struct ion_handle *entry;
 444
 445         id = idr_alloc(&client->idr, handle, 1, 0, GFP_KERNEL);
 446         if (id < 0)
 447                 return id;
 448
 449         handle->id = id;
 450
 451         while (*p) {
 452                 parent = *p;
 453                 entry = rb_entry(parent, struct ion_handle, node);
 454
 455                 if (handle->buffer < entry->buffer)
 456                         p = &(*p)->rb_left;
 457                 else if (handle->buffer > entry->buffer)
 458                         p = &(*p)->rb_right;
 459                 else
 460                         WARN(1, "%s: buffer already found.", __func__);
 461         }
 462
 463         rb_link_node(&handle->node, parent, p);
 464         rb_insert_color(&handle->node, &client->handles);
 465
 466         return 0;
 467 }
 468
 469 struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
 470                              size_t align, unsigned int heap_id_mask,
 471                              unsigned int flags)
 472 {
 473         struct ion_handle *handle;
 474         struct ion_device *dev = client->dev;
 475         struct ion_buffer *buffer = NULL;
 476         struct ion_heap *heap;
 477         int ret;
 478
 479         pr_debug("%s: len %zu align %zu heap_id_mask %u flags %x\n", __func__,
 480                  len, align, heap_id_mask, flags);
 481         /*
 482          * traverse the list of heaps available in this system in priority
 483          * order.  If the heap type is supported by the client, and matches the
 484          * request of the caller allocate from it.  Repeat until allocate has
 485          * succeeded or all heaps have been tried
 486          */
 487         len = PAGE_ALIGN(len);
 488
 489         if (!len)
 490                 return ERR_PTR(-EINVAL);
 491
 492         down_read(&dev->lock);
 493         plist_for_each_entry(heap, &dev->heaps, node) {
 494                 /* if the caller didn't specify this heap id */
 495                 if (!((1 << heap->id) & heap_id_mask))
 496                         continue;
 497                 buffer = ion_buffer_create(heap, dev, len, align, flags);
 498                 if (!IS_ERR(buffer))
 499                         break;
 500         }
 501         up_read(&dev->lock);
 502
 503         if (buffer == NULL)
 504                 return ERR_PTR(-ENODEV);
 505
 506         if (IS_ERR(buffer))
 507                 return ERR_PTR(PTR_ERR(buffer));
 508
 509         handle = ion_handle_create(client, buffer);
 510
 511         /*
 512          * ion_buffer_create will create a buffer with a ref_cnt of 1,
 513          * and ion_handle_create will take a second reference, drop one here
 514          */
 515         ion_buffer_put(buffer);
 516
 517         if (IS_ERR(handle))
 518                 return handle;
 519
 520         mutex_lock(&client->lock);
 521         ret = ion_handle_add(client, handle);
 522         mutex_unlock(&client->lock);
 523         if (ret) {
 524                 ion_handle_put(handle);
 525                 handle = ERR_PTR(ret);
 526         }
 527
 528         return handle;
 529 }
 530 EXPORT_SYMBOL(ion_alloc);
 531
 532 void ion_free(struct ion_client *client, struct ion_handle *handle)
 533 {
 534         bool valid_handle;
 535
 536         BUG_ON(client != handle->client);
 537
 538         mutex_lock(&client->lock);
 539         valid_handle = ion_handle_validate(client, handle);
 540
 541         if (!valid_handle) {
 542                 WARN(1, "%s: invalid handle passed to free.\n", __func__);
 543                 mutex_unlock(&client->lock);
 544                 return;
 545         }
 546         mutex_unlock(&client->lock);
 547         ion_handle_put(handle);
 548 }
 549 EXPORT_SYMBOL(ion_free);
 550
 551 int ion_phys(struct ion_client *client, struct ion_handle *handle,
 552              ion_phys_addr_t *addr, size_t *len)
 553 {
 554         struct ion_buffer *buffer;
 555         int ret;
 556
 557         mutex_lock(&client->lock);
 558         if (!ion_handle_validate(client, handle)) {
 559                 mutex_unlock(&client->lock);
 560                 return -EINVAL;
 561         }
 562
 563         buffer = handle->buffer;
 564
 565         if (!buffer->heap->ops->phys) {
 566                 pr_err("%s: ion_phys is not implemented by this heap.\n",
 567                        __func__);
 568                 mutex_unlock(&client->lock);
 569                 return -ENODEV;
 570         }
 571         mutex_unlock(&client->lock);
 572         ret = buffer->heap->ops->phys(buffer->heap, buffer, addr, len);
 573         return ret;
 574 }
 575 EXPORT_SYMBOL(ion_phys);
 576
 577 static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
 578 {
 579         void *vaddr;
 580
 581         if (buffer->kmap_cnt) {
 582                 buffer->kmap_cnt++;
 583                 return buffer->vaddr;
 584         }
 585         vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
 586         if (WARN_ONCE(vaddr == NULL,
 587                         "heap->ops->map_kernel should return ERR_PTR on error"))
 588                 return ERR_PTR(-EINVAL);
 589         if (IS_ERR(vaddr))
 590                 return vaddr;
 591         buffer->vaddr = vaddr;
 592         buffer->kmap_cnt++;
 593         return vaddr;
 594 }
 595
 596 static void *ion_handle_kmap_get(struct ion_handle *handle)
 597 {
 598         struct ion_buffer *buffer = handle->buffer;
 599         void *vaddr;
 600
 601         if (handle->kmap_cnt) {
 602                 handle->kmap_cnt++;
 603                 return buffer->vaddr;
 604         }
 605         vaddr = ion_buffer_kmap_get(buffer);
 606         if (IS_ERR(vaddr))
 607                 return vaddr;
 608         handle->kmap_cnt++;
 609         return vaddr;
 610 }
 611
 612 static void ion_buffer_kmap_put(struct ion_buffer *buffer)
 613 {
 614         buffer->kmap_cnt--;
 615         if (!buffer->kmap_cnt) {
 616                 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
 617                 buffer->vaddr = NULL;
 618         }
 619 }
 620
 621 static void ion_handle_kmap_put(struct ion_handle *handle)
 622 {
 623         struct ion_buffer *buffer = handle->buffer;
 624
 625         handle->kmap_cnt--;
 626         if (!handle->kmap_cnt)
 627                 ion_buffer_kmap_put(buffer);
 628 }
 629
 630 void *ion_map_kernel(struct ion_client *client, struct ion_handle *handle)
 631 {
 632         struct ion_buffer *buffer;
 633         void *vaddr;
 634
 635         mutex_lock(&client->lock);
 636         if (!ion_handle_validate(client, handle)) {
 637                 pr_err("%s: invalid handle passed to map_kernel.\n",
 638                        __func__);
 639                 mutex_unlock(&client->lock);
 640                 return ERR_PTR(-EINVAL);
 641         }
 642
 643         buffer = handle->buffer;
 644
 645         if (!handle->buffer->heap->ops->map_kernel) {
 646                 pr_err("%s: map_kernel is not implemented by this heap.\n",
 647                        __func__);
 648                 mutex_unlock(&client->lock);
 649                 return ERR_PTR(-ENODEV);
 650         }
 651
 652         mutex_lock(&buffer->lock);
 653         vaddr = ion_handle_kmap_get(handle);
 654         mutex_unlock(&buffer->lock);
 655         mutex_unlock(&client->lock);
 656         return vaddr;
 657 }
 658 EXPORT_SYMBOL(ion_map_kernel);
 659
 660 void ion_unmap_kernel(struct ion_client *client, struct ion_handle *handle)
 661 {
 662         struct ion_buffer *buffer;
 663
 664         mutex_lock(&client->lock);
 665         buffer = handle->buffer;
 666         mutex_lock(&buffer->lock);
 667         ion_handle_kmap_put(handle);
 668         mutex_unlock(&buffer->lock);
 669         mutex_unlock(&client->lock);
 670 }
 671 EXPORT_SYMBOL(ion_unmap_kernel);
 672
 673 static int ion_debug_client_show(struct seq_file *s, void *unused)
 674 {
 675         struct ion_client *client = s->private;
 676         struct rb_node *n;
 677         size_t sizes[ION_NUM_HEAP_IDS] = {0};
 678         const char *names[ION_NUM_HEAP_IDS] = {NULL};
 679         int i;
 680
 681         mutex_lock(&client->lock);
 682         for (n = rb_first(&client->handles); n; n = rb_next(n)) {
 683                 struct ion_handle *handle = rb_entry(n, struct ion_handle,
 684                                                      node);
 685                 unsigned int id = handle->buffer->heap->id;
 686
 687                 if (!names[id])
 688                         names[id] = handle->buffer->heap->name;
 689                 sizes[id] += handle->buffer->size;
 690         }
 691         mutex_unlock(&client->lock);
 692
 693         seq_printf(s, "%16.16s: %16.16s\n", "heap_name", "size_in_bytes");
 694         for (i = 0; i < ION_NUM_HEAP_IDS; i++) {
 695                 if (!names[i])
 696                         continue;
 697                 seq_printf(s, "%16.16s: %16zu\n", names[i], sizes[i]);
 698         }
 699         return 0;
 700 }
 701
 702 static int ion_debug_client_open(struct inode *inode, struct file *file)
 703 {
 704         return single_open(file, ion_debug_client_show, inode->i_private);
 705 }
 706
 707 static const struct file_operations debug_client_fops = {
 708         .open = ion_debug_client_open,
 709         .read = seq_read,
 710         .llseek = seq_lseek,
 711         .release = single_release,
 712 };
 713
 714 struct ion_client *ion_client_create(struct ion_device *dev,
 715                                      const char *name)
 716 {
 717         struct ion_client *client;
 718         struct task_struct *task;
 719         struct rb_node **p;
 720         struct rb_node *parent = NULL;
 721         struct ion_client *entry;
 722         pid_t pid;
 723
 724         get_task_struct(current->group_leader);
 725         task_lock(current->group_leader);
 726         pid = task_pid_nr(current->group_leader);
 727         /* don't bother to store task struct for kernel threads,
 728            they can't be killed anyway */
 729         if (current->group_leader->flags & PF_KTHREAD) {
 730                 put_task_struct(current->group_leader);
 731                 task = NULL;
 732         } else {
 733                 task = current->group_leader;
 734         }
 735         task_unlock(current->group_leader);
 736
 737         client = kzalloc(sizeof(struct ion_client), GFP_KERNEL);
 738         if (!client) {
 739                 if (task)
 740                         put_task_struct(current->group_leader);
 741                 return ERR_PTR(-ENOMEM);
 742         }
 743
 744         client->dev = dev;
 745         client->handles = RB_ROOT;
 746         idr_init(&client->idr);
 747         mutex_init(&client->lock);
 748         client->name = name;
 749         client->task = task;
 750         client->pid = pid;
 751
 752         down_write(&dev->lock);
 753         p = &dev->clients.rb_node;
 754         while (*p) {
 755                 parent = *p;
 756                 entry = rb_entry(parent, struct ion_client, node);
 757
 758                 if (client < entry)
 759                         p = &(*p)->rb_left;
 760                 else if (client > entry)
 761                         p = &(*p)->rb_right;
 762         }
 763         rb_link_node(&client->node, parent, p);
 764         rb_insert_color(&client->node, &dev->clients);
 765
 766         client->debug_root = debugfs_create_file(name, 0664,
 767                                                 dev->clients_debug_root,
 768                                                 client, &debug_client_fops);
 769         if (!client->debug_root) {
 770                 char buf[256], *path;
 771                 path = dentry_path(dev->clients_debug_root, buf, 256);
 772                 pr_err("Failed to create client debugfs at %s/%s\n",
 773                         path, name);
 774         }
 775
 776         up_write(&dev->lock);
 777
 778         return client;
 779 }
 780 EXPORT_SYMBOL(ion_client_create);
 781
 782 void ion_client_destroy(struct ion_client *client)
 783 {
 784         struct ion_device *dev = client->dev;
 785         struct rb_node *n;
 786
 787         pr_debug("%s: %d\n", __func__, __LINE__);
 788         while ((n = rb_first(&client->handles))) {
 789                 struct ion_handle *handle = rb_entry(n, struct ion_handle,
 790                                                      node);
 791                 ion_handle_destroy(&handle->ref);
 792         }
 793
 794         idr_destroy(&client->idr);
 795
 796         down_write(&dev->lock);
 797         if (client->task)
 798                 put_task_struct(client->task);
 799         rb_erase(&client->node, &dev->clients);
 800         debugfs_remove_recursive(client->debug_root);
 801         up_write(&dev->lock);
 802
 803         kfree(client);
 804 }
 805 EXPORT_SYMBOL(ion_client_destroy);
 806
 807 struct sg_table *ion_sg_table(struct ion_client *client,
 808                               struct ion_handle *handle)
 809 {
 810         struct ion_buffer *buffer;
 811         struct sg_table *table;
 812
 813         mutex_lock(&client->lock);
 814         if (!ion_handle_validate(client, handle)) {
 815                 pr_err("%s: invalid handle passed to map_dma.\n",
 816                        __func__);
 817                 mutex_unlock(&client->lock);
 818                 return ERR_PTR(-EINVAL);
 819         }
 820         buffer = handle->buffer;
 821         table = buffer->sg_table;
 822         mutex_unlock(&client->lock);
 823         return table;
 824 }
 825 EXPORT_SYMBOL(ion_sg_table);
 826
 827 static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
 828                                        struct device *dev,
 829                                        enum dma_data_direction direction);
 830
 831 static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
 832                                         enum dma_data_direction direction)
 833 {
 834         struct dma_buf *dmabuf = attachment->dmabuf;
 835         struct ion_buffer *buffer = dmabuf->priv;
 836
 837         ion_buffer_sync_for_device(buffer, attachment->dev, direction);
 838         return buffer->sg_table;
 839 }
 840
 841 static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
 842                               struct sg_table *table,
 843                               enum dma_data_direction direction)
 844 {
 845 }
 846
 847 void ion_pages_sync_for_device(struct device *dev, struct page *page,
 848                 size_t size, enum dma_data_direction dir)
 849 {
 850         struct scatterlist sg;
 851
 852         sg_init_table(&sg, 1);
 853         sg_set_page(&sg, page, size, 0);
 854         /*
 855          * This is not correct - sg_dma_address needs a dma_addr_t that is valid
 856          * for the the targeted device, but this works on the currently targeted
 857          * hardware.
 858          */
 859         sg_dma_address(&sg) = page_to_phys(page);
 860         dma_sync_sg_for_device(dev, &sg, 1, dir);
 861 }
 862
 863 struct ion_vma_list {
 864         struct list_head list;
 865         struct vm_area_struct *vma;
 866 };
 867
 868 static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
 869                                        struct device *dev,
 870                                        enum dma_data_direction dir)
 871 {
 872         struct ion_vma_list *vma_list;
 873         int pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
 874         int i;
 875
 876         pr_debug("%s: syncing for device %s\n", __func__,
 877                  dev ? dev_name(dev) : "null");
 878
 879         if (!ion_buffer_fault_user_mappings(buffer))
 880                 return;
 881
 882         mutex_lock(&buffer->lock);
 883         for (i = 0; i < pages; i++) {
 884                 struct page *page = buffer->pages[i];
 885
 886                 if (ion_buffer_page_is_dirty(page))
 887                         ion_pages_sync_for_device(dev, ion_buffer_page(page),
 888                                                         PAGE_SIZE, dir);
 889
 890                 ion_buffer_page_clean(buffer->pages + i);
 891         }
 892         list_for_each_entry(vma_list, &buffer->vmas, list) {
 893                 struct vm_area_struct *vma = vma_list->vma;
 894
 895                 zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start,
 896                                NULL);
 897         }
 898         mutex_unlock(&buffer->lock);
 899 }
 900
 901 static int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 902 {
 903         struct ion_buffer *buffer = vma->vm_private_data;
 904         unsigned long pfn;
 905         int ret;
 906
 907         mutex_lock(&buffer->lock);
 908         ion_buffer_page_dirty(buffer->pages + vmf->pgoff);
 909         BUG_ON(!buffer->pages || !buffer->pages[vmf->pgoff]);
 910
 911         pfn = page_to_pfn(ion_buffer_page(buffer->pages[vmf->pgoff]));
 912         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 913         mutex_unlock(&buffer->lock);
 914         if (ret)
 915                 return VM_FAULT_ERROR;
 916
 917         return VM_FAULT_NOPAGE;
 918 }
 919
 920 static void ion_vm_open(struct vm_area_struct *vma)
 921 {
 922         struct ion_buffer *buffer = vma->vm_private_data;
 923         struct ion_vma_list *vma_list;
 924
 925         vma_list = kmalloc(sizeof(struct ion_vma_list), GFP_KERNEL);
 926         if (!vma_list)
 927                 return;
 928         vma_list->vma = vma;
 929         mutex_lock(&buffer->lock);
 930         list_add(&vma_list->list, &buffer->vmas);
 931         mutex_unlock(&buffer->lock);
 932         pr_debug("%s: adding %p\n", __func__, vma);
 933 }
 934
 935 static void ion_vm_close(struct vm_area_struct *vma)
 936 {
 937         struct ion_buffer *buffer = vma->vm_private_data;
 938         struct ion_vma_list *vma_list, *tmp;
 939
 940         pr_debug("%s\n", __func__);
 941         mutex_lock(&buffer->lock);
 942         list_for_each_entry_safe(vma_list, tmp, &buffer->vmas, list) {
 943                 if (vma_list->vma != vma)
 944                         continue;
 945                 list_del(&vma_list->list);
 946                 kfree(vma_list);
 947                 pr_debug("%s: deleting %p\n", __func__, vma);
 948                 break;
 949         }
 950         mutex_unlock(&buffer->lock);
 951 }
 952
 953 static struct vm_operations_struct ion_vma_ops = {
 954         .open = ion_vm_open,
 955         .close = ion_vm_close,
 956         .fault = ion_vm_fault,
 957 };
 958
 959 static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 960 {
 961         struct ion_buffer *buffer = dmabuf->priv;
 962         int ret = 0;
 963
 964         if (!buffer->heap->ops->map_user) {
 965                 pr_err("%s: this heap does not define a method for mapping "
 966                        "to userspace\n", __func__);
 967                 return -EINVAL;
 968         }
 969
 970         if (ion_buffer_fault_user_mappings(buffer)) {
 971                 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND |
 972                                                         VM_DONTDUMP;
 973                 vma->vm_private_data = buffer;
 974                 vma->vm_ops = &ion_vma_ops;
 975                 ion_vm_open(vma);
 976                 return 0;
 977         }
 978
 979         if (!(buffer->flags & ION_FLAG_CACHED))
 980                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 981
 982         mutex_lock(&buffer->lock);
 983         /* now map it to userspace */
 984         ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
 985         mutex_unlock(&buffer->lock);
 986
 987         if (ret)
 988                 pr_err("%s: failure mapping buffer to userspace\n",
 989                        __func__);
 990
 991         return ret;
 992 }
 993
 994 static void ion_dma_buf_release(struct dma_buf *dmabuf)
 995 {
 996         struct ion_buffer *buffer = dmabuf->priv;
 997         ion_buffer_put(buffer);
 998 }
 999
1000 static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
1001 {
1002         struct ion_buffer *buffer = dmabuf->priv;
1003         return buffer->vaddr + offset * PAGE_SIZE;
1004 }
1005
1006 static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset,
1007                                void *ptr)
1008 {
1009         return;
1010 }
1011
1012 static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, size_t start,
1013                                         size_t len,
1014                                         enum dma_data_direction direction)
1015 {
1016         struct ion_buffer *buffer = dmabuf->priv;
1017         void *vaddr;
1018
1019         if (!buffer->heap->ops->map_kernel) {
1020                 pr_err("%s: map kernel is not implemented by this heap.\n",
1021                        __func__);
1022                 return -ENODEV;
1023         }
1024
1025         mutex_lock(&buffer->lock);
1026         vaddr = ion_buffer_kmap_get(buffer);
1027         mutex_unlock(&buffer->lock);
1028         return PTR_ERR_OR_ZERO(vaddr);
1029 }
1030
1031 static void ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, size_t start,
1032                                        size_t len,
1033                                        enum dma_data_direction direction)
1034 {
1035         struct ion_buffer *buffer = dmabuf->priv;
1036
1037         mutex_lock(&buffer->lock);
1038         ion_buffer_kmap_put(buffer);
1039         mutex_unlock(&buffer->lock);
1040 }
1041
1042 static struct dma_buf_ops dma_buf_ops = {
1043         .map_dma_buf = ion_map_dma_buf,
1044         .unmap_dma_buf = ion_unmap_dma_buf,
1045         .mmap = ion_mmap,
1046         .release = ion_dma_buf_release,
1047         .begin_cpu_access = ion_dma_buf_begin_cpu_access,
1048         .end_cpu_access = ion_dma_buf_end_cpu_access,
1049         .kmap_atomic = ion_dma_buf_kmap,
1050         .kunmap_atomic = ion_dma_buf_kunmap,
1051         .kmap = ion_dma_buf_kmap,
1052         .kunmap = ion_dma_buf_kunmap,
1053 };
1054
1055 struct dma_buf *ion_share_dma_buf(struct ion_client *client,
1056                                                 struct ion_handle *handle)
1057 {
1058         struct ion_buffer *buffer;
1059         struct dma_buf *dmabuf;
1060         bool valid_handle;
1061
1062         mutex_lock(&client->lock);
1063         valid_handle = ion_handle_validate(client, handle);
1064         if (!valid_handle) {
1065                 WARN(1, "%s: invalid handle passed to share.\n", __func__);
1066                 mutex_unlock(&client->lock);
1067                 return ERR_PTR(-EINVAL);
1068         }
1069         buffer = handle->buffer;
1070         ion_buffer_get(buffer);
1071         mutex_unlock(&client->lock);
1072
1073         dmabuf = dma_buf_export(buffer, &dma_buf_ops, buffer->size, O_RDWR);
1074         if (IS_ERR(dmabuf)) {
1075                 ion_buffer_put(buffer);
1076                 return dmabuf;
1077         }
1078
1079         return dmabuf;
1080 }
1081 EXPORT_SYMBOL(ion_share_dma_buf);
1082
1083 int ion_share_dma_buf_fd(struct ion_client *client, struct ion_handle *handle)
1084 {
1085         struct dma_buf *dmabuf;
1086         int fd;
1087
1088         dmabuf = ion_share_dma_buf(client, handle);
1089         if (IS_ERR(dmabuf))
1090                 return PTR_ERR(dmabuf);
1091
1092         fd = dma_buf_fd(dmabuf, O_CLOEXEC);
1093         if (fd < 0)
1094                 dma_buf_put(dmabuf);
1095
1096         return fd;
1097 }
1098 EXPORT_SYMBOL(ion_share_dma_buf_fd);
1099
1100 struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd)
1101 {
1102         struct dma_buf *dmabuf;
1103         struct ion_buffer *buffer;
1104         struct ion_handle *handle;
1105         int ret;
1106
1107         dmabuf = dma_buf_get(fd);
1108         if (IS_ERR(dmabuf))
1109                 return ERR_PTR(PTR_ERR(dmabuf));
1110         /* if this memory came from ion */
1111
1112         if (dmabuf->ops != &dma_buf_ops) {
1113                 pr_err("%s: can not import dmabuf from another exporter\n",
1114                        __func__);
1115                 dma_buf_put(dmabuf);
1116                 return ERR_PTR(-EINVAL);
1117         }
1118         buffer = dmabuf->priv;
1119
1120         mutex_lock(&client->lock);
1121         /* if a handle exists for this buffer just take a reference to it */
1122         handle = ion_handle_lookup(client, buffer);
1123         if (!IS_ERR(handle)) {
1124                 ion_handle_get(handle);
1125                 mutex_unlock(&client->lock);
1126                 goto end;
1127         }
1128         mutex_unlock(&client->lock);
1129
1130         handle = ion_handle_create(client, buffer);
1131         if (IS_ERR(handle))
1132                 goto end;
1133
1134         mutex_lock(&client->lock);
1135         ret = ion_handle_add(client, handle);
1136         mutex_unlock(&client->lock);
1137         if (ret) {
1138                 ion_handle_put(handle);
1139                 handle = ERR_PTR(ret);
1140         }
1141
1142 end:
1143         dma_buf_put(dmabuf);
1144         return handle;
1145 }
1146 EXPORT_SYMBOL(ion_import_dma_buf);
1147
1148 static int ion_sync_for_device(struct ion_client *client, int fd)
1149 {
1150         struct dma_buf *dmabuf;
1151         struct ion_buffer *buffer;
1152
1153         dmabuf = dma_buf_get(fd);
1154         if (IS_ERR(dmabuf))
1155                 return PTR_ERR(dmabuf);
1156
1157         /* if this memory came from ion */
1158         if (dmabuf->ops != &dma_buf_ops) {
1159                 pr_err("%s: can not sync dmabuf from another exporter\n",
1160                        __func__);
1161                 dma_buf_put(dmabuf);
1162                 return -EINVAL;
1163         }
1164         buffer = dmabuf->priv;
1165
1166         dma_sync_sg_for_device(NULL, buffer->sg_table->sgl,
1167                                buffer->sg_table->nents, DMA_BIDIRECTIONAL);
1168         dma_buf_put(dmabuf);
1169         return 0;
1170 }
1171
1172 /* fix up the cases where the ioctl direction bits are incorrect */
1173 static unsigned int ion_ioctl_dir(unsigned int cmd)
1174 {
1175         switch (cmd) {
1176         case ION_IOC_SYNC:
1177         case ION_IOC_FREE:
1178         case ION_IOC_CUSTOM:
1179                 return _IOC_WRITE;
1180         default:
1181                 return _IOC_DIR(cmd);
1182         }
1183 }
1184
1185 static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1186 {
1187         struct ion_client *client = filp->private_data;
1188         struct ion_device *dev = client->dev;
1189         struct ion_handle *cleanup_handle = NULL;
1190         int ret = 0;
1191         unsigned int dir;
1192
1193         union {
1194                 struct ion_fd_data fd;
1195                 struct ion_allocation_data allocation;
1196                 struct ion_handle_data handle;
1197                 struct ion_custom_data custom;
1198         } data;
1199
1200         dir = ion_ioctl_dir(cmd);
1201
1202         if (_IOC_SIZE(cmd) > sizeof(data))
1203                 return -EINVAL;
1204
1205         if (dir & _IOC_WRITE)
1206                 if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
1207                         return -EFAULT;
1208
1209         switch (cmd) {
1210         case ION_IOC_ALLOC:
1211         {
1212                 struct ion_handle *handle;
1213
1214                 handle = ion_alloc(client, data.allocation.len,
1215                                                 data.allocation.align,
1216                                                 data.allocation.heap_id_mask,
1217                                                 data.allocation.flags);
1218                 if (IS_ERR(handle))
1219                         return PTR_ERR(handle);
1220
1221                 data.allocation.handle = handle->id;
1222
1223                 cleanup_handle = handle;
1224                 break;
1225         }
1226         case ION_IOC_FREE:
1227         {
1228                 struct ion_handle *handle;
1229
1230                 handle = ion_handle_get_by_id(client, data.handle.handle);
1231                 if (IS_ERR(handle))
1232                         return PTR_ERR(handle);
1233                 ion_free(client, handle);
1234                 ion_handle_put(handle);
1235                 break;
1236         }
1237         case ION_IOC_SHARE:
1238         case ION_IOC_MAP:
1239         {
1240                 struct ion_handle *handle;
1241
1242                 handle = ion_handle_get_by_id(client, data.handle.handle);
1243                 if (IS_ERR(handle))
1244                         return PTR_ERR(handle);
1245                 data.fd.fd = ion_share_dma_buf_fd(client, handle);
1246                 ion_handle_put(handle);
1247                 if (data.fd.fd < 0)
1248                         ret = data.fd.fd;
1249                 break;
1250         }
1251         case ION_IOC_IMPORT:
1252         {
1253                 struct ion_handle *handle;
1254                 handle = ion_import_dma_buf(client, data.fd.fd);
1255                 if (IS_ERR(handle))
1256                         ret = PTR_ERR(handle);
1257                 else
1258                         data.handle.handle = handle->id;
1259                 break;
1260         }
1261         case ION_IOC_SYNC:
1262         {
1263                 ret = ion_sync_for_device(client, data.fd.fd);
1264                 break;
1265         }
1266         case ION_IOC_CUSTOM:
1267         {
1268                 if (!dev->custom_ioctl)
1269                         return -ENOTTY;
1270                 ret = dev->custom_ioctl(client, data.custom.cmd,
1271                                                 data.custom.arg);
1272                 break;
1273         }
1274         default:
1275                 return -ENOTTY;
1276         }
1277
1278         if (dir & _IOC_READ) {
1279                 if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd))) {
1280                         if (cleanup_handle)
1281                                 ion_free(client, cleanup_handle);
1282                         return -EFAULT;
1283                 }
1284         }
1285         return ret;
1286 }
1287
1288 static int ion_release(struct inode *inode, struct file *file)
1289 {
1290         struct ion_client *client = file->private_data;
1291
1292         pr_debug("%s: %d\n", __func__, __LINE__);
1293         ion_client_destroy(client);
1294         return 0;
1295 }
1296
1297 static int ion_open(struct inode *inode, struct file *file)
1298 {
1299         struct miscdevice *miscdev = file->private_data;
1300         struct ion_device *dev = container_of(miscdev, struct ion_device, dev);
1301         struct ion_client *client;
1302         char debug_name[64];
1303
1304         pr_debug("%s: %d\n", __func__, __LINE__);
1305         snprintf(debug_name, 64, "%u", task_pid_nr(current->group_leader));
1306         client = ion_client_create(dev, debug_name);
1307         if (IS_ERR(client))
1308                 return PTR_ERR(client);
1309         file->private_data = client;
1310
1311         return 0;
1312 }
1313
1314 static const struct file_operations ion_fops = {
1315         .owner          = THIS_MODULE,
1316         .open           = ion_open,
1317         .release        = ion_release,
1318         .unlocked_ioctl = ion_ioctl,
1319         .compat_ioctl   = compat_ion_ioctl,
1320 };
1321
1322 static size_t ion_debug_heap_total(struct ion_client *client,
1323                                    unsigned int id)
1324 {
1325         size_t size = 0;
1326         struct rb_node *n;
1327
1328         mutex_lock(&client->lock);
1329         for (n = rb_first(&client->handles); n; n = rb_next(n)) {
1330                 struct ion_handle *handle = rb_entry(n,
1331                                                      struct ion_handle,
1332                                                      node);
1333                 if (handle->buffer->heap->id == id)
1334                         size += handle->buffer->size;
1335         }
1336         mutex_unlock(&client->lock);
1337         return size;
1338 }
1339
1340 static int ion_debug_heap_show(struct seq_file *s, void *unused)
1341 {
1342         struct ion_heap *heap = s->private;
1343         struct ion_device *dev = heap->dev;
1344         struct rb_node *n;
1345         size_t total_size = 0;
1346         size_t total_orphaned_size = 0;
1347
1348         seq_printf(s, "%16.s %16.s %16.s\n", "client", "pid", "size");
1349         seq_printf(s, "----------------------------------------------------\n");
1350
1351         for (n = rb_first(&dev->clients); n; n = rb_next(n)) {
1352                 struct ion_client *client = rb_entry(n, struct ion_client,
1353                                                      node);
1354                 size_t size = ion_debug_heap_total(client, heap->id);
1355                 if (!size)
1356                         continue;
1357                 if (client->task) {
1358                         char task_comm[TASK_COMM_LEN];
1359
1360                         get_task_comm(task_comm, client->task);
1361                         seq_printf(s, "%16.s %16u %16zu\n", task_comm,
1362                                    client->pid, size);
1363                 } else {
1364                         seq_printf(s, "%16.s %16u %16zu\n", client->name,
1365                                    client->pid, size);
1366                 }
1367         }
1368         seq_printf(s, "----------------------------------------------------\n");
1369         seq_printf(s, "orphaned allocations (info is from last known client):"
1370                    "\n");
1371         mutex_lock(&dev->buffer_lock);
1372         for (n = rb_first(&dev->buffers); n; n = rb_next(n)) {
1373                 struct ion_buffer *buffer = rb_entry(n, struct ion_buffer,
1374                                                      node);
1375                 if (buffer->heap->id != heap->id)
1376                         continue;
1377                 total_size += buffer->size;
1378                 if (!buffer->handle_count) {
1379                         seq_printf(s, "%16.s %16u %16zu %d %d\n",
1380                                    buffer->task_comm, buffer->pid,
1381                                    buffer->size, buffer->kmap_cnt,
1382                                    atomic_read(&buffer->ref.refcount));
1383                         total_orphaned_size += buffer->size;
1384                 }
1385         }
1386         mutex_unlock(&dev->buffer_lock);
1387         seq_printf(s, "----------------------------------------------------\n");
1388         seq_printf(s, "%16.s %16zu\n", "total orphaned",
1389                    total_orphaned_size);
1390         seq_printf(s, "%16.s %16zu\n", "total ", total_size);
1391         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
1392                 seq_printf(s, "%16.s %16zu\n", "deferred free",
1393                                 heap->free_list_size);
1394         seq_printf(s, "----------------------------------------------------\n");
1395
1396         if (heap->debug_show)
1397                 heap->debug_show(heap, s, unused);
1398
1399         return 0;
1400 }
1401
1402 static int ion_debug_heap_open(struct inode *inode, struct file *file)
1403 {
1404         return single_open(file, ion_debug_heap_show, inode->i_private);
1405 }
1406
1407 static const struct file_operations debug_heap_fops = {
1408         .open = ion_debug_heap_open,
1409         .read = seq_read,
1410         .llseek = seq_lseek,
1411         .release = single_release,
1412 };
1413
1414 #ifdef DEBUG_HEAP_SHRINKER
1415 static int debug_shrink_set(void *data, u64 val)
1416 {
1417         struct ion_heap *heap = data;
1418         struct shrink_control sc;
1419         int objs;
1420
1421         sc.gfp_mask = -1;
1422         sc.nr_to_scan = 0;
1423
1424         if (!val)
1425                 return 0;
1426
1427         objs = heap->shrinker.shrink(&heap->shrinker, &sc);
1428         sc.nr_to_scan = objs;
1429
1430         heap->shrinker.shrink(&heap->shrinker, &sc);
1431         return 0;
1432 }
1433
1434 static int debug_shrink_get(void *data, u64 *val)
1435 {
1436         struct ion_heap *heap = data;
1437         struct shrink_control sc;
1438         int objs;
1439
1440         sc.gfp_mask = -1;
1441         sc.nr_to_scan = 0;
1442
1443         objs = heap->shrinker.shrink(&heap->shrinker, &sc);
1444         *val = objs;
1445         return 0;
1446 }
1447
1448 DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,
1449                         debug_shrink_set, "%llu\n");
1450 #endif
1451
1452 void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
1453 {
1454         struct dentry *debug_file;
1455
1456         if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||
1457             !heap->ops->unmap_dma)
1458                 pr_err("%s: can not add heap with invalid ops struct.\n",
1459                        __func__);
1460
1461         if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
1462                 ion_heap_init_deferred_free(heap);
1463
1464         heap->dev = dev;
1465         down_write(&dev->lock);
1466         /* use negative heap->id to reverse the priority -- when traversing
1467            the list later attempt higher id numbers first */
1468         plist_node_init(&heap->node, -heap->id);
1469         plist_add(&heap->node, &dev->heaps);
1470         debug_file = debugfs_create_file(heap->name, 0664,
1471                                         dev->heaps_debug_root, heap,
1472                                         &debug_heap_fops);
1473
1474         if (!debug_file) {
1475                 char buf[256], *path;
1476                 path = dentry_path(dev->heaps_debug_root, buf, 256);
1477                 pr_err("Failed to create heap debugfs at %s/%s\n",
1478                         path, heap->name);
1479         }
1480
1481 #ifdef DEBUG_HEAP_SHRINKER
1482         if (heap->shrinker.shrink) {
1483                 char debug_name[64];
1484
1485                 snprintf(debug_name, 64, "%s_shrink", heap->name);
1486                 debug_file = debugfs_create_file(
1487                         debug_name, 0644, dev->heaps_debug_root, heap,
1488                         &debug_shrink_fops);
1489                 if (!debug_file) {
1490                         char buf[256], *path;
1491                         path = dentry_path(dev->heaps_debug_root, buf, 256);
1492                         pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
1493                                 path, debug_name);
1494                 }
1495         }
1496 #endif
1497         up_write(&dev->lock);
1498 }
1499
1500 struct ion_device *ion_device_create(long (*custom_ioctl)
1501                                      (struct ion_client *client,
1502                                       unsigned int cmd,
1503                                       unsigned long arg))
1504 {
1505         struct ion_device *idev;
1506         int ret;
1507
1508         idev = kzalloc(sizeof(struct ion_device), GFP_KERNEL);
1509         if (!idev)
1510                 return ERR_PTR(-ENOMEM);
1511
1512         idev->dev.minor = MISC_DYNAMIC_MINOR;
1513         idev->dev.name = "ion";
1514         idev->dev.fops = &ion_fops;
1515         idev->dev.parent = NULL;
1516         ret = misc_register(&idev->dev);
1517         if (ret) {
1518                 pr_err("ion: failed to register misc device.\n");
1519                 return ERR_PTR(ret);
1520         }
1521
1522         idev->debug_root = debugfs_create_dir("ion", NULL);
1523         if (!idev->debug_root) {
1524                 pr_err("ion: failed to create debugfs root directory.\n");
1525                 goto debugfs_done;
1526         }
1527         idev->heaps_debug_root = debugfs_create_dir("heaps", idev->debug_root);
1528         if (!idev->heaps_debug_root) {
1529                 pr_err("ion: failed to create debugfs heaps directory.\n");
1530                 goto debugfs_done;
1531         }
1532         idev->clients_debug_root = debugfs_create_dir("clients",
1533                                                 idev->debug_root);
1534         if (!idev->clients_debug_root)
1535                 pr_err("ion: failed to create debugfs clients directory.\n");
1536
1537 debugfs_done:
1538
1539         idev->custom_ioctl = custom_ioctl;
1540         idev->buffers = RB_ROOT;
1541         mutex_init(&idev->buffer_lock);
1542         init_rwsem(&idev->lock);
1543         plist_head_init(&idev->heaps);
1544         idev->clients = RB_ROOT;
1545         return idev;
1546 }
1547
1548 void ion_device_destroy(struct ion_device *dev)
1549 {
1550         misc_deregister(&dev->dev);
1551         debugfs_remove_recursive(dev->debug_root);
1552         /* XXX need to free the heaps and clients ? */
1553         kfree(dev);
1554 }
1555
1556 void __init ion_reserve(struct ion_platform_data *data)
1557 {
1558         int i;
1559
1560         for (i = 0; i < data->nr; i++) {
1561                 if (data->heaps[i].size == 0)
1562                         continue;
1563
1564                 if (data->heaps[i].base == 0) {
1565                         phys_addr_t paddr;
1566                         paddr = memblock_alloc_base(data->heaps[i].size,
1567                                                     data->heaps[i].align,
1568                                                     MEMBLOCK_ALLOC_ANYWHERE);
1569                         if (!paddr) {
1570                                 pr_err("%s: error allocating memblock for heap %d\n",
1571                                         __func__, i);
1572                                 continue;
1573                         }
1574                         data->heaps[i].base = paddr;
1575                 } else {
1576                         int ret = memblock_reserve(data->heaps[i].base,
1577                                                data->heaps[i].size);
1578                         if (ret)
1579                                 pr_err("memblock reserve of %zx@%lx failed\n",
1580                                        data->heaps[i].size,
1581                                        data->heaps[i].base);
1582                 }
1583                 pr_info("%s: %s reserved base %lx size %zu\n", __func__,
1584                         data->heaps[i].name,
1585                         data->heaps[i].base,
1586                         data->heaps[i].size);
1587         }
1588 }