mm/shmem.c

   1 /*
   2  * Resizable virtual memory filesystem for Linux.
   3  *
   4  * Copyright (C) 2000 Linus Torvalds.
   5  *               2000 Transmeta Corp.
   6  *               2000-2001 Christoph Rohland
   7  *               2000-2001 SAP AG
   8  *               2002 Red Hat Inc.
   9  * Copyright (C) 2002-2011 Hugh Dickins.
  10  * Copyright (C) 2011 Google Inc.
  11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13  *
  14  * Extended attribute support for tmpfs:
  15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17  *
  18  * tiny-shmem:
  19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20  *
  21  * This file is released under the GPL.
  22  */
  23
  24 #include <linux/fs.h>
  25 #include <linux/init.h>
  26 #include <linux/vfs.h>
  27 #include <linux/mount.h>
  28 #include <linux/pagemap.h>
  29 #include <linux/file.h>
  30 #include <linux/mm.h>
  31 #include <linux/export.h>
  32 #include <linux/swap.h>
  33
  34 static struct vfsmount *shm_mnt;
  35
  36 #ifdef CONFIG_SHMEM
  37 /*
  38  * This virtual memory filesystem is heavily based on the ramfs. It
  39  * extends ramfs by the ability to use swap and honor resource limits
  40  * which makes it a completely usable filesystem.
  41  */
  42
  43 #include <linux/xattr.h>
  44 #include <linux/exportfs.h>
  45 #include <linux/posix_acl.h>
  46 #include <linux/generic_acl.h>
  47 #include <linux/mman.h>
  48 #include <linux/string.h>
  49 #include <linux/slab.h>
  50 #include <linux/backing-dev.h>
  51 #include <linux/shmem_fs.h>
  52 #include <linux/writeback.h>
  53 #include <linux/blkdev.h>
  54 #include <linux/pagevec.h>
  55 #include <linux/percpu_counter.h>
  56 #include <linux/falloc.h>
  57 #include <linux/splice.h>
  58 #include <linux/security.h>
  59 #include <linux/swapops.h>
  60 #include <linux/mempolicy.h>
  61 #include <linux/namei.h>
  62 #include <linux/ctype.h>
  63 #include <linux/migrate.h>
  64 #include <linux/highmem.h>
  65 #include <linux/seq_file.h>
  66 #include <linux/magic.h>
  67
  68 #include <asm/uaccess.h>
  69 #include <asm/pgtable.h>
  70
  71 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
  72 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
  73
  74 /* Pretend that each entry is of this size in directory's i_size */
  75 #define BOGO_DIRENT_SIZE 20
  76
  77 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  78 #define SHORT_SYMLINK_LEN 128
  79
  80 struct shmem_xattr {
  81         struct list_head list;  /* anchored by shmem_inode_info->xattr_list */
  82         char *name;             /* xattr name */
  83         size_t size;
  84         char value[0];
  85 };
  86
  87 /* Flag allocation requirements to shmem_getpage */
  88 enum sgp_type {
  89         SGP_READ,       /* don't exceed i_size, don't allocate page */
  90         SGP_CACHE,      /* don't exceed i_size, may allocate page */
  91         SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
  92         SGP_WRITE,      /* may exceed i_size, may allocate page */
  93 };
  94
  95 #ifdef CONFIG_TMPFS
  96 static unsigned long shmem_default_max_blocks(void)
  97 {
  98         return totalram_pages / 2;
  99 }
 100
 101 static unsigned long shmem_default_max_inodes(void)
 102 {
 103         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
 104 }
 105 #endif
 106
 107 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
 108 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
 109                                 struct shmem_inode_info *info, pgoff_t index);
 110 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 111         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
 112
 113 static inline int shmem_getpage(struct inode *inode, pgoff_t index,
 114         struct page **pagep, enum sgp_type sgp, int *fault_type)
 115 {
 116         return shmem_getpage_gfp(inode, index, pagep, sgp,
 117                         mapping_gfp_mask(inode->i_mapping), fault_type);
 118 }
 119
 120 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
 121 {
 122         return sb->s_fs_info;
 123 }
 124
 125 /*
 126  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 127  * for shared memory and for shared anonymous (/dev/zero) mappings
 128  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 129  * consistent with the pre-accounting of private mappings ...
 130  */
 131 static inline int shmem_acct_size(unsigned long flags, loff_t size)
 132 {
 133         return (flags & VM_NORESERVE) ?
 134                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
 135 }
 136
 137 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 138 {
 139         if (!(flags & VM_NORESERVE))
 140                 vm_unacct_memory(VM_ACCT(size));
 141 }
 142
 143 /*
 144  * ... whereas tmpfs objects are accounted incrementally as
 145  * pages are allocated, in order to allow huge sparse files.
 146  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 147  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 148  */
 149 static inline int shmem_acct_block(unsigned long flags)
 150 {
 151         return (flags & VM_NORESERVE) ?
 152                 security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
 153 }
 154
 155 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
 156 {
 157         if (flags & VM_NORESERVE)
 158                 vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
 159 }
 160
 161 static const struct super_operations shmem_ops;
 162 static const struct address_space_operations shmem_aops;
 163 static const struct file_operations shmem_file_operations;
 164 static const struct inode_operations shmem_inode_operations;
 165 static const struct inode_operations shmem_dir_inode_operations;
 166 static const struct inode_operations shmem_special_inode_operations;
 167 static const struct vm_operations_struct shmem_vm_ops;
 168
 169 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
 170         .ra_pages       = 0,    /* No readahead */
 171         .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
 172 };
 173
 174 static LIST_HEAD(shmem_swaplist);
 175 static DEFINE_MUTEX(shmem_swaplist_mutex);
 176
 177 static int shmem_reserve_inode(struct super_block *sb)
 178 {
 179         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 180         if (sbinfo->max_inodes) {
 181                 spin_lock(&sbinfo->stat_lock);
 182                 if (!sbinfo->free_inodes) {
 183                         spin_unlock(&sbinfo->stat_lock);
 184                         return -ENOSPC;
 185                 }
 186                 sbinfo->free_inodes--;
 187                 spin_unlock(&sbinfo->stat_lock);
 188         }
 189         return 0;
 190 }
 191
 192 static void shmem_free_inode(struct super_block *sb)
 193 {
 194         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 195         if (sbinfo->max_inodes) {
 196                 spin_lock(&sbinfo->stat_lock);
 197                 sbinfo->free_inodes++;
 198                 spin_unlock(&sbinfo->stat_lock);
 199         }
 200 }
 201
 202 /**
 203  * shmem_recalc_inode - recalculate the block usage of an inode
 204  * @inode: inode to recalc
 205  *
 206  * We have to calculate the free blocks since the mm can drop
 207  * undirtied hole pages behind our back.
 208  *
 209  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 210  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 211  *
 212  * It has to be called with the spinlock held.
 213  */
 214 static void shmem_recalc_inode(struct inode *inode)
 215 {
 216         struct shmem_inode_info *info = SHMEM_I(inode);
 217         long freed;
 218
 219         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
 220         if (freed > 0) {
 221                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 222                 if (sbinfo->max_blocks)
 223                         percpu_counter_add(&sbinfo->used_blocks, -freed);
 224                 info->alloced -= freed;
 225                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
 226                 shmem_unacct_blocks(info->flags, freed);
 227         }
 228 }
 229
 230 /*
 231  * Replace item expected in radix tree by a new item, while holding tree lock.
 232  */
 233 static int shmem_radix_tree_replace(struct address_space *mapping,
 234                         pgoff_t index, void *expected, void *replacement)
 235 {
 236         void **pslot;
 237         void *item = NULL;
 238
 239         VM_BUG_ON(!expected);
 240         pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
 241         if (pslot)
 242                 item = radix_tree_deref_slot_protected(pslot,
 243                                                         &mapping->tree_lock);
 244         if (item != expected)
 245                 return -ENOENT;
 246         if (replacement)
 247                 radix_tree_replace_slot(pslot, replacement);
 248         else
 249                 radix_tree_delete(&mapping->page_tree, index);
 250         return 0;
 251 }
 252
 253 /*
 254  * Like add_to_page_cache_locked, but error if expected item has gone.
 255  */
 256 static int shmem_add_to_page_cache(struct page *page,
 257                                    struct address_space *mapping,
 258                                    pgoff_t index, gfp_t gfp, void *expected)
 259 {
 260         int error = 0;
 261
 262         VM_BUG_ON(!PageLocked(page));
 263         VM_BUG_ON(!PageSwapBacked(page));
 264
 265         if (!expected)
 266                 error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
 267         if (!error) {
 268                 page_cache_get(page);
 269                 page->mapping = mapping;
 270                 page->index = index;
 271
 272                 spin_lock_irq(&mapping->tree_lock);
 273                 if (!expected)
 274                         error = radix_tree_insert(&mapping->page_tree,
 275                                                         index, page);
 276                 else
 277                         error = shmem_radix_tree_replace(mapping, index,
 278                                                         expected, page);
 279                 if (!error) {
 280                         mapping->nrpages++;
 281                         __inc_zone_page_state(page, NR_FILE_PAGES);
 282                         __inc_zone_page_state(page, NR_SHMEM);
 283                         spin_unlock_irq(&mapping->tree_lock);
 284                 } else {
 285                         page->mapping = NULL;
 286                         spin_unlock_irq(&mapping->tree_lock);
 287                         page_cache_release(page);
 288                 }
 289                 if (!expected)
 290                         radix_tree_preload_end();
 291         }
 292         if (error)
 293                 mem_cgroup_uncharge_cache_page(page);
 294         return error;
 295 }
 296
 297 /*
 298  * Like delete_from_page_cache, but substitutes swap for page.
 299  */
 300 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
 301 {
 302         struct address_space *mapping = page->mapping;
 303         int error;
 304
 305         spin_lock_irq(&mapping->tree_lock);
 306         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
 307         page->mapping = NULL;
 308         mapping->nrpages--;
 309         __dec_zone_page_state(page, NR_FILE_PAGES);
 310         __dec_zone_page_state(page, NR_SHMEM);
 311         spin_unlock_irq(&mapping->tree_lock);
 312         page_cache_release(page);
 313         BUG_ON(error);
 314 }
 315
 316 /*
 317  * Like find_get_pages, but collecting swap entries as well as pages.
 318  */
 319 static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
 320                                         pgoff_t start, unsigned int nr_pages,
 321                                         struct page **pages, pgoff_t *indices)
 322 {
 323         unsigned int i;
 324         unsigned int ret;
 325         unsigned int nr_found;
 326
 327         rcu_read_lock();
 328 restart:
 329         nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
 330                                 (void ***)pages, indices, start, nr_pages);
 331         ret = 0;
 332         for (i = 0; i < nr_found; i++) {
 333                 struct page *page;
 334 repeat:
 335                 page = radix_tree_deref_slot((void **)pages[i]);
 336                 if (unlikely(!page))
 337                         continue;
 338                 if (radix_tree_exception(page)) {
 339                         if (radix_tree_deref_retry(page))
 340                                 goto restart;
 341                         /*
 342                          * Otherwise, we must be storing a swap entry
 343                          * here as an exceptional entry: so return it
 344                          * without attempting to raise page count.
 345                          */
 346                         goto export;
 347                 }
 348                 if (!page_cache_get_speculative(page))
 349                         goto repeat;
 350
 351                 /* Has the page moved? */
 352                 if (unlikely(page != *((void **)pages[i]))) {
 353                         page_cache_release(page);
 354                         goto repeat;
 355                 }
 356 export:
 357                 indices[ret] = indices[i];
 358                 pages[ret] = page;
 359                 ret++;
 360         }
 361         if (unlikely(!ret && nr_found))
 362                 goto restart;
 363         rcu_read_unlock();
 364         return ret;
 365 }
 366
 367 /*
 368  * Remove swap entry from radix tree, free the swap and its page cache.
 369  */
 370 static int shmem_free_swap(struct address_space *mapping,
 371                            pgoff_t index, void *radswap)
 372 {
 373         int error;
 374
 375         spin_lock_irq(&mapping->tree_lock);
 376         error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
 377         spin_unlock_irq(&mapping->tree_lock);
 378         if (!error)
 379                 free_swap_and_cache(radix_to_swp_entry(radswap));
 380         return error;
 381 }
 382
 383 /*
 384  * Pagevec may contain swap entries, so shuffle up pages before releasing.
 385  */
 386 static void shmem_deswap_pagevec(struct pagevec *pvec)
 387 {
 388         int i, j;
 389
 390         for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
 391                 struct page *page = pvec->pages[i];
 392                 if (!radix_tree_exceptional_entry(page))
 393                         pvec->pages[j++] = page;
 394         }
 395         pvec->nr = j;
 396 }
 397
 398 /*
 399  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 400  */
 401 void shmem_unlock_mapping(struct address_space *mapping)
 402 {
 403         struct pagevec pvec;
 404         pgoff_t indices[PAGEVEC_SIZE];
 405         pgoff_t index = 0;
 406
 407         pagevec_init(&pvec, 0);
 408         /*
 409          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
 410          */
 411         while (!mapping_unevictable(mapping)) {
 412                 /*
 413                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
 414                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
 415                  */
 416                 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
 417                                         PAGEVEC_SIZE, pvec.pages, indices);
 418                 if (!pvec.nr)
 419                         break;
 420                 index = indices[pvec.nr - 1] + 1;
 421                 shmem_deswap_pagevec(&pvec);
 422                 check_move_unevictable_pages(pvec.pages, pvec.nr);
 423                 pagevec_release(&pvec);
 424                 cond_resched();
 425         }
 426 }
 427
 428 /*
 429  * Remove range of pages and swap entries from radix tree, and free them.
 430  */
 431 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
 432 {
 433         struct address_space *mapping = inode->i_mapping;
 434         struct shmem_inode_info *info = SHMEM_I(inode);
 435         pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 436         pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
 437         unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
 438         unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
 439         struct pagevec pvec;
 440         pgoff_t indices[PAGEVEC_SIZE];
 441         long nr_swaps_freed = 0;
 442         pgoff_t index;
 443         int i;
 444
 445         if (lend == -1)
 446                 end = -1;       /* unsigned, so actually very big */
 447
 448         pagevec_init(&pvec, 0);
 449         index = start;
 450         while (index < end) {
 451                 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
 452                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
 453                                                         pvec.pages, indices);
 454                 if (!pvec.nr)
 455                         break;
 456                 mem_cgroup_uncharge_start();
 457                 for (i = 0; i < pagevec_count(&pvec); i++) {
 458                         struct page *page = pvec.pages[i];
 459
 460                         index = indices[i];
 461                         if (index >= end)
 462                                 break;
 463
 464                         if (radix_tree_exceptional_entry(page)) {
 465                                 nr_swaps_freed += !shmem_free_swap(mapping,
 466                                                                 index, page);
 467                                 continue;
 468                         }
 469
 470                         if (!trylock_page(page))
 471                                 continue;
 472                         if (page->mapping == mapping) {
 473                                 VM_BUG_ON(PageWriteback(page));
 474                                 truncate_inode_page(mapping, page);
 475                         }
 476                         unlock_page(page);
 477                 }
 478                 shmem_deswap_pagevec(&pvec);
 479                 pagevec_release(&pvec);
 480                 mem_cgroup_uncharge_end();
 481                 cond_resched();
 482                 index++;
 483         }
 484
 485         if (partial_start) {
 486                 struct page *page = NULL;
 487                 shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
 488                 if (page) {
 489                         unsigned int top = PAGE_CACHE_SIZE;
 490                         if (start > end) {
 491                                 top = partial_end;
 492                                 partial_end = 0;
 493                         }
 494                         zero_user_segment(page, partial_start, top);
 495                         set_page_dirty(page);
 496                         unlock_page(page);
 497                         page_cache_release(page);
 498                 }
 499         }
 500         if (partial_end) {
 501                 struct page *page = NULL;
 502                 shmem_getpage(inode, end, &page, SGP_READ, NULL);
 503                 if (page) {
 504                         zero_user_segment(page, 0, partial_end);
 505                         set_page_dirty(page);
 506                         unlock_page(page);
 507                         page_cache_release(page);
 508                 }
 509         }
 510         if (start >= end)
 511                 return;
 512
 513         index = start;
 514         for ( ; ; ) {
 515                 cond_resched();
 516                 pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
 517                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
 518                                                         pvec.pages, indices);
 519                 if (!pvec.nr) {
 520                         if (index == start)
 521                                 break;
 522                         index = start;
 523                         continue;
 524                 }
 525                 if (index == start && indices[0] >= end) {
 526                         shmem_deswap_pagevec(&pvec);
 527                         pagevec_release(&pvec);
 528                         break;
 529                 }
 530                 mem_cgroup_uncharge_start();
 531                 for (i = 0; i < pagevec_count(&pvec); i++) {
 532                         struct page *page = pvec.pages[i];
 533
 534                         index = indices[i];
 535                         if (index >= end)
 536                                 break;
 537
 538                         if (radix_tree_exceptional_entry(page)) {
 539                                 nr_swaps_freed += !shmem_free_swap(mapping,
 540                                                                 index, page);
 541                                 continue;
 542                         }
 543
 544                         lock_page(page);
 545                         if (page->mapping == mapping) {
 546                                 VM_BUG_ON(PageWriteback(page));
 547                                 truncate_inode_page(mapping, page);
 548                         }
 549                         unlock_page(page);
 550                 }
 551                 shmem_deswap_pagevec(&pvec);
 552                 pagevec_release(&pvec);
 553                 mem_cgroup_uncharge_end();
 554                 index++;
 555         }
 556
 557         spin_lock(&info->lock);
 558         info->swapped -= nr_swaps_freed;
 559         shmem_recalc_inode(inode);
 560         spin_unlock(&info->lock);
 561
 562         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
 563 }
 564 EXPORT_SYMBOL_GPL(shmem_truncate_range);
 565
 566 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
 567 {
 568         struct inode *inode = dentry->d_inode;
 569         int error;
 570
 571         error = inode_change_ok(inode, attr);
 572         if (error)
 573                 return error;
 574
 575         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
 576                 loff_t oldsize = inode->i_size;
 577                 loff_t newsize = attr->ia_size;
 578
 579                 if (newsize != oldsize) {
 580                         i_size_write(inode, newsize);
 581                         inode->i_ctime = inode->i_mtime = CURRENT_TIME;
 582                 }
 583                 if (newsize < oldsize) {
 584                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
 585                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
 586                         shmem_truncate_range(inode, newsize, (loff_t)-1);
 587                         /* unmap again to remove racily COWed private pages */
 588                         unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
 589                 }
 590         }
 591
 592         setattr_copy(inode, attr);
 593 #ifdef CONFIG_TMPFS_POSIX_ACL
 594         if (attr->ia_valid & ATTR_MODE)
 595                 error = generic_acl_chmod(inode);
 596 #endif
 597         return error;
 598 }
 599
 600 static void shmem_evict_inode(struct inode *inode)
 601 {
 602         struct shmem_inode_info *info = SHMEM_I(inode);
 603         struct shmem_xattr *xattr, *nxattr;
 604
 605         if (inode->i_mapping->a_ops == &shmem_aops) {
 606                 shmem_unacct_size(info->flags, inode->i_size);
 607                 inode->i_size = 0;
 608                 shmem_truncate_range(inode, 0, (loff_t)-1);
 609                 if (!list_empty(&info->swaplist)) {
 610                         mutex_lock(&shmem_swaplist_mutex);
 611                         list_del_init(&info->swaplist);
 612                         mutex_unlock(&shmem_swaplist_mutex);
 613                 }
 614         } else
 615                 kfree(info->symlink);
 616
 617         list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
 618                 kfree(xattr->name);
 619                 kfree(xattr);
 620         }
 621         BUG_ON(inode->i_blocks);
 622         shmem_free_inode(inode->i_sb);
 623         clear_inode(inode);
 624 }
 625
 626 /*
 627  * If swap found in inode, free it and move page from swapcache to filecache.
 628  */
 629 static int shmem_unuse_inode(struct shmem_inode_info *info,
 630                              swp_entry_t swap, struct page **pagep)
 631 {
 632         struct address_space *mapping = info->vfs_inode.i_mapping;
 633         void *radswap;
 634         pgoff_t index;
 635         gfp_t gfp;
 636         int error = 0;
 637
 638         radswap = swp_to_radix_entry(swap);
 639         index = radix_tree_locate_item(&mapping->page_tree, radswap);
 640         if (index == -1)
 641                 return 0;
 642
 643         /*
 644          * Move _head_ to start search for next from here.
 645          * But be careful: shmem_evict_inode checks list_empty without taking
 646          * mutex, and there's an instant in list_move_tail when info->swaplist
 647          * would appear empty, if it were the only one on shmem_swaplist.
 648          */
 649         if (shmem_swaplist.next != &info->swaplist)
 650                 list_move_tail(&shmem_swaplist, &info->swaplist);
 651
 652         gfp = mapping_gfp_mask(mapping);
 653         if (shmem_should_replace_page(*pagep, gfp)) {
 654                 mutex_unlock(&shmem_swaplist_mutex);
 655                 error = shmem_replace_page(pagep, gfp, info, index);
 656                 mutex_lock(&shmem_swaplist_mutex);
 657                 /*
 658                  * We needed to drop mutex to make that restrictive page
 659                  * allocation; but the inode might already be freed by now,
 660                  * and we cannot refer to inode or mapping or info to check.
 661                  * However, we do hold page lock on the PageSwapCache page,
 662                  * so can check if that still has our reference remaining.
 663                  */
 664                 if (!page_swapcount(*pagep))
 665                         error = -ENOENT;
 666         }
 667
 668         /*
 669          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
 670          * but also to hold up shmem_evict_inode(): so inode cannot be freed
 671          * beneath us (pagelock doesn't help until the page is in pagecache).
 672          */
 673         if (!error)
 674                 error = shmem_add_to_page_cache(*pagep, mapping, index,
 675                                                 GFP_NOWAIT, radswap);
 676         if (error != -ENOMEM) {
 677                 /*
 678                  * Truncation and eviction use free_swap_and_cache(), which
 679                  * only does trylock page: if we raced, best clean up here.
 680                  */
 681                 delete_from_swap_cache(*pagep);
 682                 set_page_dirty(*pagep);
 683                 if (!error) {
 684                         spin_lock(&info->lock);
 685                         info->swapped--;
 686                         spin_unlock(&info->lock);
 687                         swap_free(swap);
 688                 }
 689                 error = 1;      /* not an error, but entry was found */
 690         }
 691         return error;
 692 }
 693
 694 /*
 695  * Search through swapped inodes to find and replace swap by page.
 696  */
 697 int shmem_unuse(swp_entry_t swap, struct page *page)
 698 {
 699         struct list_head *this, *next;
 700         struct shmem_inode_info *info;
 701         int found = 0;
 702         int error = 0;
 703
 704         /*
 705          * There's a faint possibility that swap page was replaced before
 706          * caller locked it: it will come back later with the right page.
 707          */
 708         if (unlikely(!PageSwapCache(page)))
 709                 goto out;
 710
 711         /*
 712          * Charge page using GFP_KERNEL while we can wait, before taking
 713          * the shmem_swaplist_mutex which might hold up shmem_writepage().
 714          * Charged back to the user (not to caller) when swap account is used.
 715          */
 716         error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
 717         if (error)
 718                 goto out;
 719         /* No radix_tree_preload: swap entry keeps a place for page in tree */
 720
 721         mutex_lock(&shmem_swaplist_mutex);
 722         list_for_each_safe(this, next, &shmem_swaplist) {
 723                 info = list_entry(this, struct shmem_inode_info, swaplist);
 724                 if (info->swapped)
 725                         found = shmem_unuse_inode(info, swap, &page);
 726                 else
 727                         list_del_init(&info->swaplist);
 728                 cond_resched();
 729                 if (found)
 730                         break;
 731         }
 732         mutex_unlock(&shmem_swaplist_mutex);
 733
 734         if (found < 0)
 735                 error = found;
 736 out:
 737         unlock_page(page);
 738         page_cache_release(page);
 739         return error;
 740 }
 741
 742 /*
 743  * Move the page from the page cache to the swap cache.
 744  */
 745 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
 746 {
 747         struct shmem_inode_info *info;
 748         struct address_space *mapping;
 749         struct inode *inode;
 750         swp_entry_t swap;
 751         pgoff_t index;
 752
 753         BUG_ON(!PageLocked(page));
 754         mapping = page->mapping;
 755         index = page->index;
 756         inode = mapping->host;
 757         info = SHMEM_I(inode);
 758         if (info->flags & VM_LOCKED)
 759                 goto redirty;
 760         if (!total_swap_pages)
 761                 goto redirty;
 762
 763         /*
 764          * shmem_backing_dev_info's capabilities prevent regular writeback or
 765          * sync from ever calling shmem_writepage; but a stacking filesystem
 766          * might use ->writepage of its underlying filesystem, in which case
 767          * tmpfs should write out to swap only in response to memory pressure,
 768          * and not for the writeback threads or sync.
 769          */
 770         if (!wbc->for_reclaim) {
 771                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
 772                 goto redirty;
 773         }
 774         swap = get_swap_page();
 775         if (!swap.val)
 776                 goto redirty;
 777
 778         /*
 779          * Add inode to shmem_unuse()'s list of swapped-out inodes,
 780          * if it's not already there.  Do it now before the page is
 781          * moved to swap cache, when its pagelock no longer protects
 782          * the inode from eviction.  But don't unlock the mutex until
 783          * we've incremented swapped, because shmem_unuse_inode() will
 784          * prune a !swapped inode from the swaplist under this mutex.
 785          */
 786         mutex_lock(&shmem_swaplist_mutex);
 787         if (list_empty(&info->swaplist))
 788                 list_add_tail(&info->swaplist, &shmem_swaplist);
 789
 790         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
 791                 swap_shmem_alloc(swap);
 792                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
 793
 794                 spin_lock(&info->lock);
 795                 info->swapped++;
 796                 shmem_recalc_inode(inode);
 797                 spin_unlock(&info->lock);
 798
 799                 mutex_unlock(&shmem_swaplist_mutex);
 800                 BUG_ON(page_mapped(page));
 801                 swap_writepage(page, wbc);
 802                 return 0;
 803         }
 804
 805         mutex_unlock(&shmem_swaplist_mutex);
 806         swapcache_free(swap, NULL);
 807 redirty:
 808         set_page_dirty(page);
 809         if (wbc->for_reclaim)
 810                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
 811         unlock_page(page);
 812         return 0;
 813 }
 814
 815 #ifdef CONFIG_NUMA
 816 #ifdef CONFIG_TMPFS
 817 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
 818 {
 819         char buffer[64];
 820
 821         if (!mpol || mpol->mode == MPOL_DEFAULT)
 822                 return;         /* show nothing */
 823
 824         mpol_to_str(buffer, sizeof(buffer), mpol, 1);
 825
 826         seq_printf(seq, ",mpol=%s", buffer);
 827 }
 828
 829 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
 830 {
 831         struct mempolicy *mpol = NULL;
 832         if (sbinfo->mpol) {
 833                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
 834                 mpol = sbinfo->mpol;
 835                 mpol_get(mpol);
 836                 spin_unlock(&sbinfo->stat_lock);
 837         }
 838         return mpol;
 839 }
 840 #endif /* CONFIG_TMPFS */
 841
 842 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
 843                         struct shmem_inode_info *info, pgoff_t index)
 844 {
 845         struct mempolicy mpol, *spol;
 846         struct vm_area_struct pvma;
 847
 848         spol = mpol_cond_copy(&mpol,
 849                         mpol_shared_policy_lookup(&info->policy, index));
 850
 851         /* Create a pseudo vma that just contains the policy */
 852         pvma.vm_start = 0;
 853         pvma.vm_pgoff = index;
 854         pvma.vm_ops = NULL;
 855         pvma.vm_policy = spol;
 856         return swapin_readahead(swap, gfp, &pvma, 0);
 857 }
 858
 859 static struct page *shmem_alloc_page(gfp_t gfp,
 860                         struct shmem_inode_info *info, pgoff_t index)
 861 {
 862         struct vm_area_struct pvma;
 863
 864         /* Create a pseudo vma that just contains the policy */
 865         pvma.vm_start = 0;
 866         pvma.vm_pgoff = index;
 867         pvma.vm_ops = NULL;
 868         pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
 869
 870         /*
 871          * alloc_page_vma() will drop the shared policy reference
 872          */
 873         return alloc_page_vma(gfp, &pvma, 0);
 874 }
 875 #else /* !CONFIG_NUMA */
 876 #ifdef CONFIG_TMPFS
 877 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
 878 {
 879 }
 880 #endif /* CONFIG_TMPFS */
 881
 882 static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
 883                         struct shmem_inode_info *info, pgoff_t index)
 884 {
 885         return swapin_readahead(swap, gfp, NULL, 0);
 886 }
 887
 888 static inline struct page *shmem_alloc_page(gfp_t gfp,
 889                         struct shmem_inode_info *info, pgoff_t index)
 890 {
 891         return alloc_page(gfp);
 892 }
 893 #endif /* CONFIG_NUMA */
 894
 895 #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
 896 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
 897 {
 898         return NULL;
 899 }
 900 #endif
 901
 902 /*
 903  * When a page is moved from swapcache to shmem filecache (either by the
 904  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
 905  * shmem_unuse_inode()), it may have been read in earlier from swap, in
 906  * ignorance of the mapping it belongs to.  If that mapping has special
 907  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
 908  * we may need to copy to a suitable page before moving to filecache.
 909  *
 910  * In a future release, this may well be extended to respect cpuset and
 911  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
 912  * but for now it is a simple matter of zone.
 913  */
 914 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
 915 {
 916         return page_zonenum(page) > gfp_zone(gfp);
 917 }
 918
 919 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
 920                                 struct shmem_inode_info *info, pgoff_t index)
 921 {
 922         struct page *oldpage, *newpage;
 923         struct address_space *swap_mapping;
 924         pgoff_t swap_index;
 925         int error;
 926
 927         oldpage = *pagep;
 928         swap_index = page_private(oldpage);
 929         swap_mapping = page_mapping(oldpage);
 930
 931         /*
 932          * We have arrived here because our zones are constrained, so don't
 933          * limit chance of success by further cpuset and node constraints.
 934          */
 935         gfp &= ~GFP_CONSTRAINT_MASK;
 936         newpage = shmem_alloc_page(gfp, info, index);
 937         if (!newpage)
 938                 return -ENOMEM;
 939         VM_BUG_ON(shmem_should_replace_page(newpage, gfp));
 940
 941         *pagep = newpage;
 942         page_cache_get(newpage);
 943         copy_highpage(newpage, oldpage);
 944
 945         VM_BUG_ON(!PageLocked(oldpage));
 946         __set_page_locked(newpage);
 947         VM_BUG_ON(!PageUptodate(oldpage));
 948         SetPageUptodate(newpage);
 949         VM_BUG_ON(!PageSwapBacked(oldpage));
 950         SetPageSwapBacked(newpage);
 951         VM_BUG_ON(!swap_index);
 952         set_page_private(newpage, swap_index);
 953         VM_BUG_ON(!PageSwapCache(oldpage));
 954         SetPageSwapCache(newpage);
 955
 956         /*
 957          * Our caller will very soon move newpage out of swapcache, but it's
 958          * a nice clean interface for us to replace oldpage by newpage there.
 959          */
 960         spin_lock_irq(&swap_mapping->tree_lock);
 961         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
 962                                                                    newpage);
 963         __inc_zone_page_state(newpage, NR_FILE_PAGES);
 964         __dec_zone_page_state(oldpage, NR_FILE_PAGES);
 965         spin_unlock_irq(&swap_mapping->tree_lock);
 966         BUG_ON(error);
 967
 968         mem_cgroup_replace_page_cache(oldpage, newpage);
 969         lru_cache_add_anon(newpage);
 970
 971         ClearPageSwapCache(oldpage);
 972         set_page_private(oldpage, 0);
 973
 974         unlock_page(oldpage);
 975         page_cache_release(oldpage);
 976         page_cache_release(oldpage);
 977         return 0;
 978 }
 979
 980 /*
 981  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
 982  *
 983  * If we allocate a new one we do not mark it dirty. That's up to the
 984  * vm. If we swap it in we mark it dirty since we also free the swap
 985  * entry since a page cannot live in both the swap and page cache
 986  */
 987 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 988         struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
 989 {
 990         struct address_space *mapping = inode->i_mapping;
 991         struct shmem_inode_info *info;
 992         struct shmem_sb_info *sbinfo;
 993         struct page *page;
 994         swp_entry_t swap;
 995         int error;
 996         int once = 0;
 997
 998         if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
 999                 return -EFBIG;
1000 repeat:
1001         swap.val = 0;
1002         page = find_lock_page(mapping, index);
1003         if (radix_tree_exceptional_entry(page)) {
1004                 swap = radix_to_swp_entry(page);
1005                 page = NULL;
1006         }
1007
1008         if (sgp != SGP_WRITE &&
1009             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1010                 error = -EINVAL;
1011                 goto failed;
1012         }
1013
1014         if (page || (sgp == SGP_READ && !swap.val)) {
1015                 /*
1016                  * Once we can get the page lock, it must be uptodate:
1017                  * if there were an error in reading back from swap,
1018                  * the page would not be inserted into the filecache.
1019                  */
1020                 BUG_ON(page && !PageUptodate(page));
1021                 *pagep = page;
1022                 return 0;
1023         }
1024
1025         /*
1026          * Fast cache lookup did not find it:
1027          * bring it back from swap or allocate.
1028          */
1029         info = SHMEM_I(inode);
1030         sbinfo = SHMEM_SB(inode->i_sb);
1031
1032         if (swap.val) {
1033                 /* Look it up and read it in.. */
1034                 page = lookup_swap_cache(swap);
1035                 if (!page) {
1036                         /* here we actually do the io */
1037                         if (fault_type)
1038                                 *fault_type |= VM_FAULT_MAJOR;
1039                         page = shmem_swapin(swap, gfp, info, index);
1040                         if (!page) {
1041                                 error = -ENOMEM;
1042                                 goto failed;
1043                         }
1044                 }
1045
1046                 /* We have to do this with page locked to prevent races */
1047                 lock_page(page);
1048                 if (!PageSwapCache(page) || page->mapping) {
1049                         error = -EEXIST;        /* try again */
1050                         goto failed;
1051                 }
1052                 if (!PageUptodate(page)) {
1053                         error = -EIO;
1054                         goto failed;
1055                 }
1056                 wait_on_page_writeback(page);
1057
1058                 if (shmem_should_replace_page(page, gfp)) {
1059                         error = shmem_replace_page(&page, gfp, info, index);
1060                         if (error)
1061                                 goto failed;
1062                 }
1063
1064                 error = mem_cgroup_cache_charge(page, current->mm,
1065                                                 gfp & GFP_RECLAIM_MASK);
1066                 if (!error)
1067                         error = shmem_add_to_page_cache(page, mapping, index,
1068                                                 gfp, swp_to_radix_entry(swap));
1069                 if (error)
1070                         goto failed;
1071
1072                 spin_lock(&info->lock);
1073                 info->swapped--;
1074                 shmem_recalc_inode(inode);
1075                 spin_unlock(&info->lock);
1076
1077                 delete_from_swap_cache(page);
1078                 set_page_dirty(page);
1079                 swap_free(swap);
1080
1081         } else {
1082                 if (shmem_acct_block(info->flags)) {
1083                         error = -ENOSPC;
1084                         goto failed;
1085                 }
1086                 if (sbinfo->max_blocks) {
1087                         if (percpu_counter_compare(&sbinfo->used_blocks,
1088                                                 sbinfo->max_blocks) >= 0) {
1089                                 error = -ENOSPC;
1090                                 goto unacct;
1091                         }
1092                         percpu_counter_inc(&sbinfo->used_blocks);
1093                 }
1094
1095                 page = shmem_alloc_page(gfp, info, index);
1096                 if (!page) {
1097                         error = -ENOMEM;
1098                         goto decused;
1099                 }
1100
1101                 SetPageSwapBacked(page);
1102                 __set_page_locked(page);
1103                 error = mem_cgroup_cache_charge(page, current->mm,
1104                                                 gfp & GFP_RECLAIM_MASK);
1105                 if (!error)
1106                         error = shmem_add_to_page_cache(page, mapping, index,
1107                                                 gfp, NULL);
1108                 if (error)
1109                         goto decused;
1110                 lru_cache_add_anon(page);
1111
1112                 spin_lock(&info->lock);
1113                 info->alloced++;
1114                 inode->i_blocks += BLOCKS_PER_PAGE;
1115                 shmem_recalc_inode(inode);
1116                 spin_unlock(&info->lock);
1117
1118                 /*
1119                  * Let SGP_WRITE caller clear ends if write does not fill page
1120                  */
1121                 if (sgp != SGP_WRITE) {
1122                         clear_highpage(page);
1123                         flush_dcache_page(page);
1124                         SetPageUptodate(page);
1125                 }
1126                 if (sgp == SGP_DIRTY)
1127                         set_page_dirty(page);
1128         }
1129
1130         /* Perhaps the file has been truncated since we checked */
1131         if (sgp != SGP_WRITE &&
1132             ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1133                 error = -EINVAL;
1134                 goto trunc;
1135         }
1136         *pagep = page;
1137         return 0;
1138
1139         /*
1140          * Error recovery.
1141          */
1142 trunc:
1143         ClearPageDirty(page);
1144         delete_from_page_cache(page);
1145         spin_lock(&info->lock);
1146         info->alloced--;
1147         inode->i_blocks -= BLOCKS_PER_PAGE;
1148         spin_unlock(&info->lock);
1149 decused:
1150         if (sbinfo->max_blocks)
1151                 percpu_counter_add(&sbinfo->used_blocks, -1);
1152 unacct:
1153         shmem_unacct_blocks(info->flags, 1);
1154 failed:
1155         if (swap.val && error != -EINVAL) {
1156                 struct page *test = find_get_page(mapping, index);
1157                 if (test && !radix_tree_exceptional_entry(test))
1158                         page_cache_release(test);
1159                 /* Have another try if the entry has changed */
1160                 if (test != swp_to_radix_entry(swap))
1161                         error = -EEXIST;
1162         }
1163         if (page) {
1164                 unlock_page(page);
1165                 page_cache_release(page);
1166         }
1167         if (error == -ENOSPC && !once++) {
1168                 info = SHMEM_I(inode);
1169                 spin_lock(&info->lock);
1170                 shmem_recalc_inode(inode);
1171                 spin_unlock(&info->lock);
1172                 goto repeat;
1173         }
1174         if (error == -EEXIST)
1175                 goto repeat;
1176         return error;
1177 }
1178
1179 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1180 {
1181         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1182         int error;
1183         int ret = VM_FAULT_LOCKED;
1184
1185         error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1186         if (error)
1187                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1188
1189         if (ret & VM_FAULT_MAJOR) {
1190                 count_vm_event(PGMAJFAULT);
1191                 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1192         }
1193         return ret;
1194 }
1195
1196 #ifdef CONFIG_NUMA
1197 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
1198 {
1199         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1200         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1201 }
1202
1203 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1204                                           unsigned long addr)
1205 {
1206         struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1207         pgoff_t index;
1208
1209         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1210         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1211 }
1212 #endif
1213
1214 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1215 {
1216         struct inode *inode = file->f_path.dentry->d_inode;
1217         struct shmem_inode_info *info = SHMEM_I(inode);
1218         int retval = -ENOMEM;
1219
1220         spin_lock(&info->lock);
1221         if (lock && !(info->flags & VM_LOCKED)) {
1222                 if (!user_shm_lock(inode->i_size, user))
1223                         goto out_nomem;
1224                 info->flags |= VM_LOCKED;
1225                 mapping_set_unevictable(file->f_mapping);
1226         }
1227         if (!lock && (info->flags & VM_LOCKED) && user) {
1228                 user_shm_unlock(inode->i_size, user);
1229                 info->flags &= ~VM_LOCKED;
1230                 mapping_clear_unevictable(file->f_mapping);
1231         }
1232         retval = 0;
1233
1234 out_nomem:
1235         spin_unlock(&info->lock);
1236         return retval;
1237 }
1238
1239 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1240 {
1241         file_accessed(file);
1242         vma->vm_ops = &shmem_vm_ops;
1243         vma->vm_flags |= VM_CAN_NONLINEAR;
1244         return 0;
1245 }
1246
1247 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1248                                      umode_t mode, dev_t dev, unsigned long flags)
1249 {
1250         struct inode *inode;
1251         struct shmem_inode_info *info;
1252         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1253
1254         if (shmem_reserve_inode(sb))
1255                 return NULL;
1256
1257         inode = new_inode(sb);
1258         if (inode) {
1259                 inode->i_ino = get_next_ino();
1260                 inode_init_owner(inode, dir, mode);
1261                 inode->i_blocks = 0;
1262                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1263                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1264                 inode->i_generation = get_seconds();
1265                 info = SHMEM_I(inode);
1266                 memset(info, 0, (char *)inode - (char *)info);
1267                 spin_lock_init(&info->lock);
1268                 info->flags = flags & VM_NORESERVE;
1269                 INIT_LIST_HEAD(&info->swaplist);
1270                 INIT_LIST_HEAD(&info->xattr_list);
1271                 cache_no_acl(inode);
1272
1273                 switch (mode & S_IFMT) {
1274                 default:
1275                         inode->i_op = &shmem_special_inode_operations;
1276                         init_special_inode(inode, mode, dev);
1277                         break;
1278                 case S_IFREG:
1279                         inode->i_mapping->a_ops = &shmem_aops;
1280                         inode->i_op = &shmem_inode_operations;
1281                         inode->i_fop = &shmem_file_operations;
1282                         mpol_shared_policy_init(&info->policy,
1283                                                  shmem_get_sbmpol(sbinfo));
1284                         break;
1285                 case S_IFDIR:
1286                         inc_nlink(inode);
1287                         /* Some things misbehave if size == 0 on a directory */
1288                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
1289                         inode->i_op = &shmem_dir_inode_operations;
1290                         inode->i_fop = &simple_dir_operations;
1291                         break;
1292                 case S_IFLNK:
1293                         /*
1294                          * Must not load anything in the rbtree,
1295                          * mpol_free_shared_policy will not be called.
1296                          */
1297                         mpol_shared_policy_init(&info->policy, NULL);
1298                         break;
1299                 }
1300         } else
1301                 shmem_free_inode(sb);
1302         return inode;
1303 }
1304
1305 #ifdef CONFIG_TMPFS
1306 static const struct inode_operations shmem_symlink_inode_operations;
1307 static const struct inode_operations shmem_short_symlink_operations;
1308
1309 #ifdef CONFIG_TMPFS_XATTR
1310 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
1311 #else
1312 #define shmem_initxattrs NULL
1313 #endif
1314
1315 static int
1316 shmem_write_begin(struct file *file, struct address_space *mapping,
1317                         loff_t pos, unsigned len, unsigned flags,
1318                         struct page **pagep, void **fsdata)
1319 {
1320         struct inode *inode = mapping->host;
1321         pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1322         return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1323 }
1324
1325 static int
1326 shmem_write_end(struct file *file, struct address_space *mapping,
1327                         loff_t pos, unsigned len, unsigned copied,
1328                         struct page *page, void *fsdata)
1329 {
1330         struct inode *inode = mapping->host;
1331
1332         if (pos + copied > inode->i_size)
1333                 i_size_write(inode, pos + copied);
1334
1335         if (!PageUptodate(page)) {
1336                 if (copied < PAGE_CACHE_SIZE) {
1337                         unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1338                         zero_user_segments(page, 0, from,
1339                                         from + copied, PAGE_CACHE_SIZE);
1340                 }
1341                 SetPageUptodate(page);
1342         }
1343         set_page_dirty(page);
1344         unlock_page(page);
1345         page_cache_release(page);
1346
1347         return copied;
1348 }
1349
1350 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1351 {
1352         struct inode *inode = filp->f_path.dentry->d_inode;
1353         struct address_space *mapping = inode->i_mapping;
1354         pgoff_t index;
1355         unsigned long offset;
1356         enum sgp_type sgp = SGP_READ;
1357
1358         /*
1359          * Might this read be for a stacking filesystem?  Then when reading
1360          * holes of a sparse file, we actually need to allocate those pages,
1361          * and even mark them dirty, so it cannot exceed the max_blocks limit.
1362          */
1363         if (segment_eq(get_fs(), KERNEL_DS))
1364                 sgp = SGP_DIRTY;
1365
1366         index = *ppos >> PAGE_CACHE_SHIFT;
1367         offset = *ppos & ~PAGE_CACHE_MASK;
1368
1369         for (;;) {
1370                 struct page *page = NULL;
1371                 pgoff_t end_index;
1372                 unsigned long nr, ret;
1373                 loff_t i_size = i_size_read(inode);
1374
1375                 end_index = i_size >> PAGE_CACHE_SHIFT;
1376                 if (index > end_index)
1377                         break;
1378                 if (index == end_index) {
1379                         nr = i_size & ~PAGE_CACHE_MASK;
1380                         if (nr <= offset)
1381                                 break;
1382                 }
1383
1384                 desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1385                 if (desc->error) {
1386                         if (desc->error == -EINVAL)
1387                                 desc->error = 0;
1388                         break;
1389                 }
1390                 if (page)
1391                         unlock_page(page);
1392
1393                 /*
1394                  * We must evaluate after, since reads (unlike writes)
1395                  * are called without i_mutex protection against truncate
1396                  */
1397                 nr = PAGE_CACHE_SIZE;
1398                 i_size = i_size_read(inode);
1399                 end_index = i_size >> PAGE_CACHE_SHIFT;
1400                 if (index == end_index) {
1401                         nr = i_size & ~PAGE_CACHE_MASK;
1402                         if (nr <= offset) {
1403                                 if (page)
1404                                         page_cache_release(page);
1405                                 break;
1406                         }
1407                 }
1408                 nr -= offset;
1409
1410                 if (page) {
1411                         /*
1412                          * If users can be writing to this page using arbitrary
1413                          * virtual addresses, take care about potential aliasing
1414                          * before reading the page on the kernel side.
1415                          */
1416                         if (mapping_writably_mapped(mapping))
1417                                 flush_dcache_page(page);
1418                         /*
1419                          * Mark the page accessed if we read the beginning.
1420                          */
1421                         if (!offset)
1422                                 mark_page_accessed(page);
1423                 } else {
1424                         page = ZERO_PAGE(0);
1425                         page_cache_get(page);
1426                 }
1427
1428                 /*
1429                  * Ok, we have the page, and it's up-to-date, so
1430                  * now we can copy it to user space...
1431                  *
1432                  * The actor routine returns how many bytes were actually used..
1433                  * NOTE! This may not be the same as how much of a user buffer
1434                  * we filled up (we may be padding etc), so we can only update
1435                  * "pos" here (the actor routine has to update the user buffer
1436                  * pointers and the remaining count).
1437                  */
1438                 ret = actor(desc, page, offset, nr);
1439                 offset += ret;
1440                 index += offset >> PAGE_CACHE_SHIFT;
1441                 offset &= ~PAGE_CACHE_MASK;
1442
1443                 page_cache_release(page);
1444                 if (ret != nr || !desc->count)
1445                         break;
1446
1447                 cond_resched();
1448         }
1449
1450         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1451         file_accessed(filp);
1452 }
1453
1454 static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1455                 const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1456 {
1457         struct file *filp = iocb->ki_filp;
1458         ssize_t retval;
1459         unsigned long seg;
1460         size_t count;
1461         loff_t *ppos = &iocb->ki_pos;
1462
1463         retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1464         if (retval)
1465                 return retval;
1466
1467         for (seg = 0; seg < nr_segs; seg++) {
1468                 read_descriptor_t desc;
1469
1470                 desc.written = 0;
1471                 desc.arg.buf = iov[seg].iov_base;
1472                 desc.count = iov[seg].iov_len;
1473                 if (desc.count == 0)
1474                         continue;
1475                 desc.error = 0;
1476                 do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1477                 retval += desc.written;
1478                 if (desc.error) {
1479                         retval = retval ?: desc.error;
1480                         break;
1481                 }
1482                 if (desc.count > 0)
1483                         break;
1484         }
1485         return retval;
1486 }
1487
1488 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
1489                                 struct pipe_inode_info *pipe, size_t len,
1490                                 unsigned int flags)
1491 {
1492         struct address_space *mapping = in->f_mapping;
1493         struct inode *inode = mapping->host;
1494         unsigned int loff, nr_pages, req_pages;
1495         struct page *pages[PIPE_DEF_BUFFERS];
1496         struct partial_page partial[PIPE_DEF_BUFFERS];
1497         struct page *page;
1498         pgoff_t index, end_index;
1499         loff_t isize, left;
1500         int error, page_nr;
1501         struct splice_pipe_desc spd = {
1502                 .pages = pages,
1503                 .partial = partial,
1504                 .flags = flags,
1505                 .ops = &page_cache_pipe_buf_ops,
1506                 .spd_release = spd_release_page,
1507         };
1508
1509         isize = i_size_read(inode);
1510         if (unlikely(*ppos >= isize))
1511                 return 0;
1512
1513         left = isize - *ppos;
1514         if (unlikely(left < len))
1515                 len = left;
1516
1517         if (splice_grow_spd(pipe, &spd))
1518                 return -ENOMEM;
1519
1520         index = *ppos >> PAGE_CACHE_SHIFT;
1521         loff = *ppos & ~PAGE_CACHE_MASK;
1522         req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1523         nr_pages = min(req_pages, pipe->buffers);
1524
1525         spd.nr_pages = find_get_pages_contig(mapping, index,
1526                                                 nr_pages, spd.pages);
1527         index += spd.nr_pages;
1528         error = 0;
1529
1530         while (spd.nr_pages < nr_pages) {
1531                 error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1532                 if (error)
1533                         break;
1534                 unlock_page(page);
1535                 spd.pages[spd.nr_pages++] = page;
1536                 index++;
1537         }
1538
1539         index = *ppos >> PAGE_CACHE_SHIFT;
1540         nr_pages = spd.nr_pages;
1541         spd.nr_pages = 0;
1542
1543         for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1544                 unsigned int this_len;
1545
1546                 if (!len)
1547                         break;
1548
1549                 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1550                 page = spd.pages[page_nr];
1551
1552                 if (!PageUptodate(page) || page->mapping != mapping) {
1553                         error = shmem_getpage(inode, index, &page,
1554                                                         SGP_CACHE, NULL);
1555                         if (error)
1556                                 break;
1557                         unlock_page(page);
1558                         page_cache_release(spd.pages[page_nr]);
1559                         spd.pages[page_nr] = page;
1560                 }
1561
1562                 isize = i_size_read(inode);
1563                 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1564                 if (unlikely(!isize || index > end_index))
1565                         break;
1566
1567                 if (end_index == index) {
1568                         unsigned int plen;
1569
1570                         plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1571                         if (plen <= loff)
1572                                 break;
1573
1574                         this_len = min(this_len, plen - loff);
1575                         len = this_len;
1576                 }
1577
1578                 spd.partial[page_nr].offset = loff;
1579                 spd.partial[page_nr].len = this_len;
1580                 len -= this_len;
1581                 loff = 0;
1582                 spd.nr_pages++;
1583                 index++;
1584         }
1585
1586         while (page_nr < nr_pages)
1587                 page_cache_release(spd.pages[page_nr++]);
1588
1589         if (spd.nr_pages)
1590                 error = splice_to_pipe(pipe, &spd);
1591
1592         splice_shrink_spd(pipe, &spd);
1593
1594         if (error > 0) {
1595                 *ppos += error;
1596                 file_accessed(in);
1597         }
1598         return error;
1599 }
1600
1601 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
1602                                                          loff_t len)
1603 {
1604         struct inode *inode = file->f_path.dentry->d_inode;
1605         int error = -EOPNOTSUPP;
1606
1607         mutex_lock(&inode->i_mutex);
1608
1609         if (mode & FALLOC_FL_PUNCH_HOLE) {
1610                 struct address_space *mapping = file->f_mapping;
1611                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
1612                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
1613
1614                 if ((u64)unmap_end > (u64)unmap_start)
1615                         unmap_mapping_range(mapping, unmap_start,
1616                                             1 + unmap_end - unmap_start, 0);
1617                 shmem_truncate_range(inode, offset, offset + len - 1);
1618                 /* No need to unmap again: hole-punching leaves COWed pages */
1619                 error = 0;
1620         }
1621
1622         mutex_unlock(&inode->i_mutex);
1623         return error;
1624 }
1625
1626 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1627 {
1628         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1629
1630         buf->f_type = TMPFS_MAGIC;
1631         buf->f_bsize = PAGE_CACHE_SIZE;
1632         buf->f_namelen = NAME_MAX;
1633         if (sbinfo->max_blocks) {
1634                 buf->f_blocks = sbinfo->max_blocks;
1635                 buf->f_bavail =
1636                 buf->f_bfree  = sbinfo->max_blocks -
1637                                 percpu_counter_sum(&sbinfo->used_blocks);
1638         }
1639         if (sbinfo->max_inodes) {
1640                 buf->f_files = sbinfo->max_inodes;
1641                 buf->f_ffree = sbinfo->free_inodes;
1642         }
1643         /* else leave those fields 0 like simple_statfs */
1644         return 0;
1645 }
1646
1647 /*
1648  * File creation. Allocate an inode, and we're done..
1649  */
1650 static int
1651 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
1652 {
1653         struct inode *inode;
1654         int error = -ENOSPC;
1655
1656         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1657         if (inode) {
1658                 error = security_inode_init_security(inode, dir,
1659                                                      &dentry->d_name,
1660                                                      shmem_initxattrs, NULL);
1661                 if (error) {
1662                         if (error != -EOPNOTSUPP) {
1663                                 iput(inode);
1664                                 return error;
1665                         }
1666                 }
1667 #ifdef CONFIG_TMPFS_POSIX_ACL
1668                 error = generic_acl_init(inode, dir);
1669                 if (error) {
1670                         iput(inode);
1671                         return error;
1672                 }
1673 #else
1674                 error = 0;
1675 #endif
1676                 dir->i_size += BOGO_DIRENT_SIZE;
1677                 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1678                 d_instantiate(dentry, inode);
1679                 dget(dentry); /* Extra count - pin the dentry in core */
1680         }
1681         return error;
1682 }
1683
1684 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
1685 {
1686         int error;
1687
1688         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1689                 return error;
1690         inc_nlink(dir);
1691         return 0;
1692 }
1693
1694 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
1695                 struct nameidata *nd)
1696 {
1697         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1698 }
1699
1700 /*
1701  * Link a file..
1702  */
1703 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1704 {
1705         struct inode *inode = old_dentry->d_inode;
1706         int ret;
1707
1708         /*
1709          * No ordinary (disk based) filesystem counts links as inodes;
1710          * but each new link needs a new dentry, pinning lowmem, and
1711          * tmpfs dentries cannot be pruned until they are unlinked.
1712          */
1713         ret = shmem_reserve_inode(inode->i_sb);
1714         if (ret)
1715                 goto out;
1716
1717         dir->i_size += BOGO_DIRENT_SIZE;
1718         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1719         inc_nlink(inode);
1720         ihold(inode);   /* New dentry reference */
1721         dget(dentry);           /* Extra pinning count for the created dentry */
1722         d_instantiate(dentry, inode);
1723 out:
1724         return ret;
1725 }
1726
1727 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1728 {
1729         struct inode *inode = dentry->d_inode;
1730
1731         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1732                 shmem_free_inode(inode->i_sb);
1733
1734         dir->i_size -= BOGO_DIRENT_SIZE;
1735         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1736         drop_nlink(inode);
1737         dput(dentry);   /* Undo the count from "create" - this does all the work */
1738         return 0;
1739 }
1740
1741 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1742 {
1743         if (!simple_empty(dentry))
1744                 return -ENOTEMPTY;
1745
1746         drop_nlink(dentry->d_inode);
1747         drop_nlink(dir);
1748         return shmem_unlink(dir, dentry);
1749 }
1750
1751 /*
1752  * The VFS layer already does all the dentry stuff for rename,
1753  * we just have to decrement the usage count for the target if
1754  * it exists so that the VFS layer correctly free's it when it
1755  * gets overwritten.
1756  */
1757 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1758 {
1759         struct inode *inode = old_dentry->d_inode;
1760         int they_are_dirs = S_ISDIR(inode->i_mode);
1761
1762         if (!simple_empty(new_dentry))
1763                 return -ENOTEMPTY;
1764
1765         if (new_dentry->d_inode) {
1766                 (void) shmem_unlink(new_dir, new_dentry);
1767                 if (they_are_dirs)
1768                         drop_nlink(old_dir);
1769         } else if (they_are_dirs) {
1770                 drop_nlink(old_dir);
1771                 inc_nlink(new_dir);
1772         }
1773
1774         old_dir->i_size -= BOGO_DIRENT_SIZE;
1775         new_dir->i_size += BOGO_DIRENT_SIZE;
1776         old_dir->i_ctime = old_dir->i_mtime =
1777         new_dir->i_ctime = new_dir->i_mtime =
1778         inode->i_ctime = CURRENT_TIME;
1779         return 0;
1780 }
1781
1782 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1783 {
1784         int error;
1785         int len;
1786         struct inode *inode;
1787         struct page *page;
1788         char *kaddr;
1789         struct shmem_inode_info *info;
1790
1791         len = strlen(symname) + 1;
1792         if (len > PAGE_CACHE_SIZE)
1793                 return -ENAMETOOLONG;
1794
1795         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1796         if (!inode)
1797                 return -ENOSPC;
1798
1799         error = security_inode_init_security(inode, dir, &dentry->d_name,
1800                                              shmem_initxattrs, NULL);
1801         if (error) {
1802                 if (error != -EOPNOTSUPP) {
1803                         iput(inode);
1804                         return error;
1805                 }
1806                 error = 0;
1807         }
1808
1809         info = SHMEM_I(inode);
1810         inode->i_size = len-1;
1811         if (len <= SHORT_SYMLINK_LEN) {
1812                 info->symlink = kmemdup(symname, len, GFP_KERNEL);
1813                 if (!info->symlink) {
1814                         iput(inode);
1815                         return -ENOMEM;
1816                 }
1817                 inode->i_op = &shmem_short_symlink_operations;
1818         } else {
1819                 error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1820                 if (error) {
1821                         iput(inode);
1822                         return error;
1823                 }
1824                 inode->i_mapping->a_ops = &shmem_aops;
1825                 inode->i_op = &shmem_symlink_inode_operations;
1826                 kaddr = kmap_atomic(page);
1827                 memcpy(kaddr, symname, len);
1828                 kunmap_atomic(kaddr);
1829                 SetPageUptodate(page);
1830                 set_page_dirty(page);
1831                 unlock_page(page);
1832                 page_cache_release(page);
1833         }
1834         dir->i_size += BOGO_DIRENT_SIZE;
1835         dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1836         d_instantiate(dentry, inode);
1837         dget(dentry);
1838         return 0;
1839 }
1840
1841 static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
1842 {
1843         nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
1844         return NULL;
1845 }
1846
1847 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1848 {
1849         struct page *page = NULL;
1850         int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1851         nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
1852         if (page)
1853                 unlock_page(page);
1854         return page;
1855 }
1856
1857 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1858 {
1859         if (!IS_ERR(nd_get_link(nd))) {
1860                 struct page *page = cookie;
1861                 kunmap(page);
1862                 mark_page_accessed(page);
1863                 page_cache_release(page);
1864         }
1865 }
1866
1867 #ifdef CONFIG_TMPFS_XATTR
1868 /*
1869  * Superblocks without xattr inode operations may get some security.* xattr
1870  * support from the LSM "for free". As soon as we have any other xattrs
1871  * like ACLs, we also need to implement the security.* handlers at
1872  * filesystem level, though.
1873  */
1874
1875 /*
1876  * Allocate new xattr and copy in the value; but leave the name to callers.
1877  */
1878 static struct shmem_xattr *shmem_xattr_alloc(const void *value, size_t size)
1879 {
1880         struct shmem_xattr *new_xattr;
1881         size_t len;
1882
1883         /* wrap around? */
1884         len = sizeof(*new_xattr) + size;
1885         if (len <= sizeof(*new_xattr))
1886                 return NULL;
1887
1888         new_xattr = kmalloc(len, GFP_KERNEL);
1889         if (!new_xattr)
1890                 return NULL;
1891
1892         new_xattr->size = size;
1893         memcpy(new_xattr->value, value, size);
1894         return new_xattr;
1895 }
1896
1897 /*
1898  * Callback for security_inode_init_security() for acquiring xattrs.
1899  */
1900 static int shmem_initxattrs(struct inode *inode,
1901                             const struct xattr *xattr_array,
1902                             void *fs_info)
1903 {
1904         struct shmem_inode_info *info = SHMEM_I(inode);
1905         const struct xattr *xattr;
1906         struct shmem_xattr *new_xattr;
1907         size_t len;
1908
1909         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
1910                 new_xattr = shmem_xattr_alloc(xattr->value, xattr->value_len);
1911                 if (!new_xattr)
1912                         return -ENOMEM;
1913
1914                 len = strlen(xattr->name) + 1;
1915                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
1916                                           GFP_KERNEL);
1917                 if (!new_xattr->name) {
1918                         kfree(new_xattr);
1919                         return -ENOMEM;
1920                 }
1921
1922                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
1923                        XATTR_SECURITY_PREFIX_LEN);
1924                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
1925                        xattr->name, len);
1926
1927                 spin_lock(&info->lock);
1928                 list_add(&new_xattr->list, &info->xattr_list);
1929                 spin_unlock(&info->lock);
1930         }
1931
1932         return 0;
1933 }
1934
1935 static int shmem_xattr_get(struct dentry *dentry, const char *name,
1936                            void *buffer, size_t size)
1937 {
1938         struct shmem_inode_info *info;
1939         struct shmem_xattr *xattr;
1940         int ret = -ENODATA;
1941
1942         info = SHMEM_I(dentry->d_inode);
1943
1944         spin_lock(&info->lock);
1945         list_for_each_entry(xattr, &info->xattr_list, list) {
1946                 if (strcmp(name, xattr->name))
1947                         continue;
1948
1949                 ret = xattr->size;
1950                 if (buffer) {
1951                         if (size < xattr->size)
1952                                 ret = -ERANGE;
1953                         else
1954                                 memcpy(buffer, xattr->value, xattr->size);
1955                 }
1956                 break;
1957         }
1958         spin_unlock(&info->lock);
1959         return ret;
1960 }
1961
1962 static int shmem_xattr_set(struct inode *inode, const char *name,
1963                            const void *value, size_t size, int flags)
1964 {
1965         struct shmem_inode_info *info = SHMEM_I(inode);
1966         struct shmem_xattr *xattr;
1967         struct shmem_xattr *new_xattr = NULL;
1968         int err = 0;
1969
1970         /* value == NULL means remove */
1971         if (value) {
1972                 new_xattr = shmem_xattr_alloc(value, size);
1973                 if (!new_xattr)
1974                         return -ENOMEM;
1975
1976                 new_xattr->name = kstrdup(name, GFP_KERNEL);
1977                 if (!new_xattr->name) {
1978                         kfree(new_xattr);
1979                         return -ENOMEM;
1980                 }
1981         }
1982
1983         spin_lock(&info->lock);
1984         list_for_each_entry(xattr, &info->xattr_list, list) {
1985                 if (!strcmp(name, xattr->name)) {
1986                         if (flags & XATTR_CREATE) {
1987                                 xattr = new_xattr;
1988                                 err = -EEXIST;
1989                         } else if (new_xattr) {
1990                                 list_replace(&xattr->list, &new_xattr->list);
1991                         } else {
1992                                 list_del(&xattr->list);
1993                         }
1994                         goto out;
1995                 }
1996         }
1997         if (flags & XATTR_REPLACE) {
1998                 xattr = new_xattr;
1999                 err = -ENODATA;
2000         } else {
2001                 list_add(&new_xattr->list, &info->xattr_list);
2002                 xattr = NULL;
2003         }
2004 out:
2005         spin_unlock(&info->lock);
2006         if (xattr)
2007                 kfree(xattr->name);
2008         kfree(xattr);
2009         return err;
2010 }
2011
2012 static const struct xattr_handler *shmem_xattr_handlers[] = {
2013 #ifdef CONFIG_TMPFS_POSIX_ACL
2014         &generic_acl_access_handler,
2015         &generic_acl_default_handler,
2016 #endif
2017         NULL
2018 };
2019
2020 static int shmem_xattr_validate(const char *name)
2021 {
2022         struct { const char *prefix; size_t len; } arr[] = {
2023                 { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
2024                 { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
2025         };
2026         int i;
2027
2028         for (i = 0; i < ARRAY_SIZE(arr); i++) {
2029                 size_t preflen = arr[i].len;
2030                 if (strncmp(name, arr[i].prefix, preflen) == 0) {
2031                         if (!name[preflen])
2032                                 return -EINVAL;
2033                         return 0;
2034                 }
2035         }
2036         return -EOPNOTSUPP;
2037 }
2038
2039 static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
2040                               void *buffer, size_t size)
2041 {
2042         int err;
2043
2044         /*
2045          * If this is a request for a synthetic attribute in the system.*
2046          * namespace use the generic infrastructure to resolve a handler
2047          * for it via sb->s_xattr.
2048          */
2049         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2050                 return generic_getxattr(dentry, name, buffer, size);
2051
2052         err = shmem_xattr_validate(name);
2053         if (err)
2054                 return err;
2055
2056         return shmem_xattr_get(dentry, name, buffer, size);
2057 }
2058
2059 static int shmem_setxattr(struct dentry *dentry, const char *name,
2060                           const void *value, size_t size, int flags)
2061 {
2062         int err;
2063
2064         /*
2065          * If this is a request for a synthetic attribute in the system.*
2066          * namespace use the generic infrastructure to resolve a handler
2067          * for it via sb->s_xattr.
2068          */
2069         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2070                 return generic_setxattr(dentry, name, value, size, flags);
2071
2072         err = shmem_xattr_validate(name);
2073         if (err)
2074                 return err;
2075
2076         if (size == 0)
2077                 value = "";  /* empty EA, do not remove */
2078
2079         return shmem_xattr_set(dentry->d_inode, name, value, size, flags);
2080
2081 }
2082
2083 static int shmem_removexattr(struct dentry *dentry, const char *name)
2084 {
2085         int err;
2086
2087         /*
2088          * If this is a request for a synthetic attribute in the system.*
2089          * namespace use the generic infrastructure to resolve a handler
2090          * for it via sb->s_xattr.
2091          */
2092         if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2093                 return generic_removexattr(dentry, name);
2094
2095         err = shmem_xattr_validate(name);
2096         if (err)
2097                 return err;
2098
2099         return shmem_xattr_set(dentry->d_inode, name, NULL, 0, XATTR_REPLACE);
2100 }
2101
2102 static bool xattr_is_trusted(const char *name)
2103 {
2104         return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
2105 }
2106
2107 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
2108 {
2109         bool trusted = capable(CAP_SYS_ADMIN);
2110         struct shmem_xattr *xattr;
2111         struct shmem_inode_info *info;
2112         size_t used = 0;
2113
2114         info = SHMEM_I(dentry->d_inode);
2115
2116         spin_lock(&info->lock);
2117         list_for_each_entry(xattr, &info->xattr_list, list) {
2118                 size_t len;
2119
2120                 /* skip "trusted." attributes for unprivileged callers */
2121                 if (!trusted && xattr_is_trusted(xattr->name))
2122                         continue;
2123
2124                 len = strlen(xattr->name) + 1;
2125                 used += len;
2126                 if (buffer) {
2127                         if (size < used) {
2128                                 used = -ERANGE;
2129                                 break;
2130                         }
2131                         memcpy(buffer, xattr->name, len);
2132                         buffer += len;
2133                 }
2134         }
2135         spin_unlock(&info->lock);
2136
2137         return used;
2138 }
2139 #endif /* CONFIG_TMPFS_XATTR */
2140
2141 static const struct inode_operations shmem_short_symlink_operations = {
2142         .readlink       = generic_readlink,
2143         .follow_link    = shmem_follow_short_symlink,
2144 #ifdef CONFIG_TMPFS_XATTR
2145         .setxattr       = shmem_setxattr,
2146         .getxattr       = shmem_getxattr,
2147         .listxattr      = shmem_listxattr,
2148         .removexattr    = shmem_removexattr,
2149 #endif
2150 };
2151
2152 static const struct inode_operations shmem_symlink_inode_operations = {
2153         .readlink       = generic_readlink,
2154         .follow_link    = shmem_follow_link,
2155         .put_link       = shmem_put_link,
2156 #ifdef CONFIG_TMPFS_XATTR
2157         .setxattr       = shmem_setxattr,
2158         .getxattr       = shmem_getxattr,
2159         .listxattr      = shmem_listxattr,
2160         .removexattr    = shmem_removexattr,
2161 #endif
2162 };
2163
2164 static struct dentry *shmem_get_parent(struct dentry *child)
2165 {
2166         return ERR_PTR(-ESTALE);
2167 }
2168
2169 static int shmem_match(struct inode *ino, void *vfh)
2170 {
2171         __u32 *fh = vfh;
2172         __u64 inum = fh[2];
2173         inum = (inum << 32) | fh[1];
2174         return ino->i_ino == inum && fh[0] == ino->i_generation;
2175 }
2176
2177 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2178                 struct fid *fid, int fh_len, int fh_type)
2179 {
2180         struct inode *inode;
2181         struct dentry *dentry = NULL;
2182         u64 inum = fid->raw[2];
2183         inum = (inum << 32) | fid->raw[1];
2184
2185         if (fh_len < 3)
2186                 return NULL;
2187
2188         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2189                         shmem_match, fid->raw);
2190         if (inode) {
2191                 dentry = d_find_alias(inode);
2192                 iput(inode);
2193         }
2194
2195         return dentry;
2196 }
2197
2198 static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2199                                 int connectable)
2200 {
2201         struct inode *inode = dentry->d_inode;
2202
2203         if (*len < 3) {
2204                 *len = 3;
2205                 return 255;
2206         }
2207
2208         if (inode_unhashed(inode)) {
2209                 /* Unfortunately insert_inode_hash is not idempotent,
2210                  * so as we hash inodes here rather than at creation
2211                  * time, we need a lock to ensure we only try
2212                  * to do it once
2213                  */
2214                 static DEFINE_SPINLOCK(lock);
2215                 spin_lock(&lock);
2216                 if (inode_unhashed(inode))
2217                         __insert_inode_hash(inode,
2218                                             inode->i_ino + inode->i_generation);
2219                 spin_unlock(&lock);
2220         }
2221
2222         fh[0] = inode->i_generation;
2223         fh[1] = inode->i_ino;
2224         fh[2] = ((__u64)inode->i_ino) >> 32;
2225
2226         *len = 3;
2227         return 1;
2228 }
2229
2230 static const struct export_operations shmem_export_ops = {
2231         .get_parent     = shmem_get_parent,
2232         .encode_fh      = shmem_encode_fh,
2233         .fh_to_dentry   = shmem_fh_to_dentry,
2234 };
2235
2236 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2237                                bool remount)
2238 {
2239         char *this_char, *value, *rest;
2240         uid_t uid;
2241         gid_t gid;
2242
2243         while (options != NULL) {
2244                 this_char = options;
2245                 for (;;) {
2246                         /*
2247                          * NUL-terminate this option: unfortunately,
2248                          * mount options form a comma-separated list,
2249                          * but mpol's nodelist may also contain commas.
2250                          */
2251                         options = strchr(options, ',');
2252                         if (options == NULL)
2253                                 break;
2254                         options++;
2255                         if (!isdigit(*options)) {
2256                                 options[-1] = '\0';
2257                                 break;
2258                         }
2259                 }
2260                 if (!*this_char)
2261                         continue;
2262                 if ((value = strchr(this_char,'=')) != NULL) {
2263                         *value++ = 0;
2264                 } else {
2265                         printk(KERN_ERR
2266                             "tmpfs: No value for mount option '%s'\n",
2267                             this_char);
2268                         return 1;
2269                 }
2270
2271                 if (!strcmp(this_char,"size")) {
2272                         unsigned long long size;
2273                         size = memparse(value,&rest);
2274                         if (*rest == '%') {
2275                                 size <<= PAGE_SHIFT;
2276                                 size *= totalram_pages;
2277                                 do_div(size, 100);
2278                                 rest++;
2279                         }
2280                         if (*rest)
2281                                 goto bad_val;
2282                         sbinfo->max_blocks =
2283                                 DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2284                 } else if (!strcmp(this_char,"nr_blocks")) {
2285                         sbinfo->max_blocks = memparse(value, &rest);
2286                         if (*rest)
2287                                 goto bad_val;
2288                 } else if (!strcmp(this_char,"nr_inodes")) {
2289                         sbinfo->max_inodes = memparse(value, &rest);
2290                         if (*rest)
2291                                 goto bad_val;
2292                 } else if (!strcmp(this_char,"mode")) {
2293                         if (remount)
2294                                 continue;
2295                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2296                         if (*rest)
2297                                 goto bad_val;
2298                 } else if (!strcmp(this_char,"uid")) {
2299                         if (remount)
2300                                 continue;
2301                         uid = simple_strtoul(value, &rest, 0);
2302                         if (*rest)
2303                                 goto bad_val;
2304                         sbinfo->uid = make_kuid(current_user_ns(), uid);
2305                         if (!uid_valid(sbinfo->uid))
2306                                 goto bad_val;
2307                 } else if (!strcmp(this_char,"gid")) {
2308                         if (remount)
2309                                 continue;
2310                         gid = simple_strtoul(value, &rest, 0);
2311                         if (*rest)
2312                                 goto bad_val;
2313                         sbinfo->gid = make_kgid(current_user_ns(), gid);
2314                         if (!gid_valid(sbinfo->gid))
2315                                 goto bad_val;
2316                 } else if (!strcmp(this_char,"mpol")) {
2317                         if (mpol_parse_str(value, &sbinfo->mpol, 1))
2318                                 goto bad_val;
2319                 } else {
2320                         printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2321                                this_char);
2322                         return 1;
2323                 }
2324         }
2325         return 0;
2326
2327 bad_val:
2328         printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2329                value, this_char);
2330         return 1;
2331
2332 }
2333
2334 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2335 {
2336         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2337         struct shmem_sb_info config = *sbinfo;
2338         unsigned long inodes;
2339         int error = -EINVAL;
2340
2341         if (shmem_parse_options(data, &config, true))
2342                 return error;
2343
2344         spin_lock(&sbinfo->stat_lock);
2345         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2346         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2347                 goto out;
2348         if (config.max_inodes < inodes)
2349                 goto out;
2350         /*
2351          * Those tests disallow limited->unlimited while any are in use;
2352          * but we must separately disallow unlimited->limited, because
2353          * in that case we have no record of how much is already in use.
2354          */
2355         if (config.max_blocks && !sbinfo->max_blocks)
2356                 goto out;
2357         if (config.max_inodes && !sbinfo->max_inodes)
2358                 goto out;
2359
2360         error = 0;
2361         sbinfo->max_blocks  = config.max_blocks;
2362         sbinfo->max_inodes  = config.max_inodes;
2363         sbinfo->free_inodes = config.max_inodes - inodes;
2364
2365         mpol_put(sbinfo->mpol);
2366         sbinfo->mpol        = config.mpol;      /* transfers initial ref */
2367 out:
2368         spin_unlock(&sbinfo->stat_lock);
2369         return error;
2370 }
2371
2372 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
2373 {
2374         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
2375
2376         if (sbinfo->max_blocks != shmem_default_max_blocks())
2377                 seq_printf(seq, ",size=%luk",
2378                         sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2379         if (sbinfo->max_inodes != shmem_default_max_inodes())
2380                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2381         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2382                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
2383         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2384                 seq_printf(seq, ",uid=%u",
2385                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
2386         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2387                 seq_printf(seq, ",gid=%u",
2388                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
2389         shmem_show_mpol(seq, sbinfo->mpol);
2390         return 0;
2391 }
2392 #endif /* CONFIG_TMPFS */
2393
2394 static void shmem_put_super(struct super_block *sb)
2395 {
2396         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2397
2398         percpu_counter_destroy(&sbinfo->used_blocks);
2399         kfree(sbinfo);
2400         sb->s_fs_info = NULL;
2401 }
2402
2403 int shmem_fill_super(struct super_block *sb, void *data, int silent)
2404 {
2405         struct inode *inode;
2406         struct shmem_sb_info *sbinfo;
2407         int err = -ENOMEM;
2408
2409         /* Round up to L1_CACHE_BYTES to resist false sharing */
2410         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2411                                 L1_CACHE_BYTES), GFP_KERNEL);
2412         if (!sbinfo)
2413                 return -ENOMEM;
2414
2415         sbinfo->mode = S_IRWXUGO | S_ISVTX;
2416         sbinfo->uid = current_fsuid();
2417         sbinfo->gid = current_fsgid();
2418         sb->s_fs_info = sbinfo;
2419
2420 #ifdef CONFIG_TMPFS
2421         /*
2422          * Per default we only allow half of the physical ram per
2423          * tmpfs instance, limiting inodes to one per page of lowmem;
2424          * but the internal instance is left unlimited.
2425          */
2426         if (!(sb->s_flags & MS_NOUSER)) {
2427                 sbinfo->max_blocks = shmem_default_max_blocks();
2428                 sbinfo->max_inodes = shmem_default_max_inodes();
2429                 if (shmem_parse_options(data, sbinfo, false)) {
2430                         err = -EINVAL;
2431                         goto failed;
2432                 }
2433         }
2434         sb->s_export_op = &shmem_export_ops;
2435         sb->s_flags |= MS_NOSEC;
2436 #else
2437         sb->s_flags |= MS_NOUSER;
2438 #endif
2439
2440         spin_lock_init(&sbinfo->stat_lock);
2441         if (percpu_counter_init(&sbinfo->used_blocks, 0))
2442                 goto failed;
2443         sbinfo->free_inodes = sbinfo->max_inodes;
2444
2445         sb->s_maxbytes = MAX_LFS_FILESIZE;
2446         sb->s_blocksize = PAGE_CACHE_SIZE;
2447         sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2448         sb->s_magic = TMPFS_MAGIC;
2449         sb->s_op = &shmem_ops;
2450         sb->s_time_gran = 1;
2451 #ifdef CONFIG_TMPFS_XATTR
2452         sb->s_xattr = shmem_xattr_handlers;
2453 #endif
2454 #ifdef CONFIG_TMPFS_POSIX_ACL
2455         sb->s_flags |= MS_POSIXACL;
2456 #endif
2457
2458         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2459         if (!inode)
2460                 goto failed;
2461         inode->i_uid = sbinfo->uid;
2462         inode->i_gid = sbinfo->gid;
2463         sb->s_root = d_make_root(inode);
2464         if (!sb->s_root)
2465                 goto failed;
2466         return 0;
2467
2468 failed:
2469         shmem_put_super(sb);
2470         return err;
2471 }
2472
2473 static struct kmem_cache *shmem_inode_cachep;
2474
2475 static struct inode *shmem_alloc_inode(struct super_block *sb)
2476 {
2477         struct shmem_inode_info *info;
2478         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2479         if (!info)
2480                 return NULL;
2481         return &info->vfs_inode;
2482 }
2483
2484 static void shmem_destroy_callback(struct rcu_head *head)
2485 {
2486         struct inode *inode = container_of(head, struct inode, i_rcu);
2487         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2488 }
2489
2490 static void shmem_destroy_inode(struct inode *inode)
2491 {
2492         if (S_ISREG(inode->i_mode))
2493                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2494         call_rcu(&inode->i_rcu, shmem_destroy_callback);
2495 }
2496
2497 static void shmem_init_inode(void *foo)
2498 {
2499         struct shmem_inode_info *info = foo;
2500         inode_init_once(&info->vfs_inode);
2501 }
2502
2503 static int shmem_init_inodecache(void)
2504 {
2505         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2506                                 sizeof(struct shmem_inode_info),
2507                                 0, SLAB_PANIC, shmem_init_inode);
2508         return 0;
2509 }
2510
2511 static void shmem_destroy_inodecache(void)
2512 {
2513         kmem_cache_destroy(shmem_inode_cachep);
2514 }
2515
2516 static const struct address_space_operations shmem_aops = {
2517         .writepage      = shmem_writepage,
2518         .set_page_dirty = __set_page_dirty_no_writeback,
2519 #ifdef CONFIG_TMPFS
2520         .write_begin    = shmem_write_begin,
2521         .write_end      = shmem_write_end,
2522 #endif
2523         .migratepage    = migrate_page,
2524         .error_remove_page = generic_error_remove_page,
2525 };
2526
2527 static const struct file_operations shmem_file_operations = {
2528         .mmap           = shmem_mmap,
2529 #ifdef CONFIG_TMPFS
2530         .llseek         = generic_file_llseek,
2531         .read           = do_sync_read,
2532         .write          = do_sync_write,
2533         .aio_read       = shmem_file_aio_read,
2534         .aio_write      = generic_file_aio_write,
2535         .fsync          = noop_fsync,
2536         .splice_read    = shmem_file_splice_read,
2537         .splice_write   = generic_file_splice_write,
2538         .fallocate      = shmem_fallocate,
2539 #endif
2540 };
2541
2542 static const struct inode_operations shmem_inode_operations = {
2543         .setattr        = shmem_setattr,
2544 #ifdef CONFIG_TMPFS_XATTR
2545         .setxattr       = shmem_setxattr,
2546         .getxattr       = shmem_getxattr,
2547         .listxattr      = shmem_listxattr,
2548         .removexattr    = shmem_removexattr,
2549 #endif
2550 };
2551
2552 static const struct inode_operations shmem_dir_inode_operations = {
2553 #ifdef CONFIG_TMPFS
2554         .create         = shmem_create,
2555         .lookup         = simple_lookup,
2556         .link           = shmem_link,
2557         .unlink         = shmem_unlink,
2558         .symlink        = shmem_symlink,
2559         .mkdir          = shmem_mkdir,
2560         .rmdir          = shmem_rmdir,
2561         .mknod          = shmem_mknod,
2562         .rename         = shmem_rename,
2563 #endif
2564 #ifdef CONFIG_TMPFS_XATTR
2565         .setxattr       = shmem_setxattr,
2566         .getxattr       = shmem_getxattr,
2567         .listxattr      = shmem_listxattr,
2568         .removexattr    = shmem_removexattr,
2569 #endif
2570 #ifdef CONFIG_TMPFS_POSIX_ACL
2571         .setattr        = shmem_setattr,
2572 #endif
2573 };
2574
2575 static const struct inode_operations shmem_special_inode_operations = {
2576 #ifdef CONFIG_TMPFS_XATTR
2577         .setxattr       = shmem_setxattr,
2578         .getxattr       = shmem_getxattr,
2579         .listxattr      = shmem_listxattr,
2580         .removexattr    = shmem_removexattr,
2581 #endif
2582 #ifdef CONFIG_TMPFS_POSIX_ACL
2583         .setattr        = shmem_setattr,
2584 #endif
2585 };
2586
2587 static const struct super_operations shmem_ops = {
2588         .alloc_inode    = shmem_alloc_inode,
2589         .destroy_inode  = shmem_destroy_inode,
2590 #ifdef CONFIG_TMPFS
2591         .statfs         = shmem_statfs,
2592         .remount_fs     = shmem_remount_fs,
2593         .show_options   = shmem_show_options,
2594 #endif
2595         .evict_inode    = shmem_evict_inode,
2596         .drop_inode     = generic_delete_inode,
2597         .put_super      = shmem_put_super,
2598 };
2599
2600 static const struct vm_operations_struct shmem_vm_ops = {
2601         .fault          = shmem_fault,
2602 #ifdef CONFIG_NUMA
2603         .set_policy     = shmem_set_policy,
2604         .get_policy     = shmem_get_policy,
2605 #endif
2606 };
2607
2608 static struct dentry *shmem_mount(struct file_system_type *fs_type,
2609         int flags, const char *dev_name, void *data)
2610 {
2611         return mount_nodev(fs_type, flags, data, shmem_fill_super);
2612 }
2613
2614 static struct file_system_type shmem_fs_type = {
2615         .owner          = THIS_MODULE,
2616         .name           = "tmpfs",
2617         .mount          = shmem_mount,
2618         .kill_sb        = kill_litter_super,
2619 };
2620
2621 int __init shmem_init(void)
2622 {
2623         int error;
2624
2625         error = bdi_init(&shmem_backing_dev_info);
2626         if (error)
2627                 goto out4;
2628
2629         error = shmem_init_inodecache();
2630         if (error)
2631                 goto out3;
2632
2633         error = register_filesystem(&shmem_fs_type);
2634         if (error) {
2635                 printk(KERN_ERR "Could not register tmpfs\n");
2636                 goto out2;
2637         }
2638
2639         shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
2640                                  shmem_fs_type.name, NULL);
2641         if (IS_ERR(shm_mnt)) {
2642                 error = PTR_ERR(shm_mnt);
2643                 printk(KERN_ERR "Could not kern_mount tmpfs\n");
2644                 goto out1;
2645         }
2646         return 0;
2647
2648 out1:
2649         unregister_filesystem(&shmem_fs_type);
2650 out2:
2651         shmem_destroy_inodecache();
2652 out3:
2653         bdi_destroy(&shmem_backing_dev_info);
2654 out4:
2655         shm_mnt = ERR_PTR(error);
2656         return error;
2657 }
2658
2659 #else /* !CONFIG_SHMEM */
2660
2661 /*
2662  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2663  *
2664  * This is intended for small system where the benefits of the full
2665  * shmem code (swap-backed and resource-limited) are outweighed by
2666  * their complexity. On systems without swap this code should be
2667  * effectively equivalent, but much lighter weight.
2668  */
2669
2670 #include <linux/ramfs.h>
2671
2672 static struct file_system_type shmem_fs_type = {
2673         .name           = "tmpfs",
2674         .mount          = ramfs_mount,
2675         .kill_sb        = kill_litter_super,
2676 };
2677
2678 int __init shmem_init(void)
2679 {
2680         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2681
2682         shm_mnt = kern_mount(&shmem_fs_type);
2683         BUG_ON(IS_ERR(shm_mnt));
2684
2685         return 0;
2686 }
2687
2688 int shmem_unuse(swp_entry_t swap, struct page *page)
2689 {
2690         return 0;
2691 }
2692
2693 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2694 {
2695         return 0;
2696 }
2697
2698 void shmem_unlock_mapping(struct address_space *mapping)
2699 {
2700 }
2701
2702 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2703 {
2704         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2705 }
2706 EXPORT_SYMBOL_GPL(shmem_truncate_range);
2707
2708 #define shmem_vm_ops                            generic_file_vm_ops
2709 #define shmem_file_operations                   ramfs_file_operations
2710 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
2711 #define shmem_acct_size(flags, size)            0
2712 #define shmem_unacct_size(flags, size)          do {} while (0)
2713
2714 #endif /* CONFIG_SHMEM */
2715
2716 /* common code */
2717
2718 /**
2719  * shmem_file_setup - get an unlinked file living in tmpfs
2720  * @name: name for dentry (to be seen in /proc/<pid>/maps
2721  * @size: size to be set for the file
2722  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2723  */
2724 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2725 {
2726         int error;
2727         struct file *file;
2728         struct inode *inode;
2729         struct path path;
2730         struct dentry *root;
2731         struct qstr this;
2732
2733         if (IS_ERR(shm_mnt))
2734                 return (void *)shm_mnt;
2735
2736         if (size < 0 || size > MAX_LFS_FILESIZE)
2737                 return ERR_PTR(-EINVAL);
2738
2739         if (shmem_acct_size(flags, size))
2740                 return ERR_PTR(-ENOMEM);
2741
2742         error = -ENOMEM;
2743         this.name = name;
2744         this.len = strlen(name);
2745         this.hash = 0; /* will go */
2746         root = shm_mnt->mnt_root;
2747         path.dentry = d_alloc(root, &this);
2748         if (!path.dentry)
2749                 goto put_memory;
2750         path.mnt = mntget(shm_mnt);
2751
2752         error = -ENOSPC;
2753         inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2754         if (!inode)
2755                 goto put_dentry;
2756
2757         d_instantiate(path.dentry, inode);
2758         inode->i_size = size;
2759         clear_nlink(inode);     /* It is unlinked */
2760 #ifndef CONFIG_MMU
2761         error = ramfs_nommu_expand_for_mapping(inode, size);
2762         if (error)
2763                 goto put_dentry;
2764 #endif
2765
2766         error = -ENFILE;
2767         file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2768                   &shmem_file_operations);
2769         if (!file)
2770                 goto put_dentry;
2771
2772         return file;
2773
2774 put_dentry:
2775         path_put(&path);
2776 put_memory:
2777         shmem_unacct_size(flags, size);
2778         return ERR_PTR(error);
2779 }
2780 EXPORT_SYMBOL_GPL(shmem_file_setup);
2781
2782 /**
2783  * shmem_zero_setup - setup a shared anonymous mapping
2784  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2785  */
2786 int shmem_zero_setup(struct vm_area_struct *vma)
2787 {
2788         struct file *file;
2789         loff_t size = vma->vm_end - vma->vm_start;
2790
2791         file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2792         if (IS_ERR(file))
2793                 return PTR_ERR(file);
2794
2795         if (vma->vm_file)
2796                 fput(vma->vm_file);
2797         vma->vm_file = file;
2798         vma->vm_ops = &shmem_vm_ops;
2799         vma->vm_flags |= VM_CAN_NONLINEAR;
2800         return 0;
2801 }
2802
2803 /**
2804  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
2805  * @mapping:    the page's address_space
2806  * @index:      the page index
2807  * @gfp:        the page allocator flags to use if allocating
2808  *
2809  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
2810  * with any new page allocations done using the specified allocation flags.
2811  * But read_cache_page_gfp() uses the ->readpage() method: which does not
2812  * suit tmpfs, since it may have pages in swapcache, and needs to find those
2813  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
2814  *
2815  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
2816  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
2817  */
2818 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
2819                                          pgoff_t index, gfp_t gfp)
2820 {
2821 #ifdef CONFIG_SHMEM
2822         struct inode *inode = mapping->host;
2823         struct page *page;
2824         int error;
2825
2826         BUG_ON(mapping->a_ops != &shmem_aops);
2827         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
2828         if (error)
2829                 page = ERR_PTR(error);
2830         else
2831                 unlock_page(page);
2832         return page;
2833 #else
2834         /*
2835          * The tiny !SHMEM case uses ramfs without swap
2836          */
2837         return read_cache_page_gfp(mapping, index, gfp);
2838 #endif
2839 }
2840 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);