fs/btrfs/extent_io.c

   1 #include <linux/bitops.h>
   2 #include <linux/slab.h>
   3 #include <linux/bio.h>
   4 #include <linux/mm.h>
   5 #include <linux/pagemap.h>
   6 #include <linux/page-flags.h>
   7 #include <linux/spinlock.h>
   8 #include <linux/blkdev.h>
   9 #include <linux/swap.h>
  10 #include <linux/writeback.h>
  11 #include <linux/pagevec.h>
  12 #include <linux/prefetch.h>
  13 #include <linux/cleancache.h>
  14 #include "extent_io.h"
  15 #include "extent_map.h"
  16 #include "compat.h"
  17 #include "ctree.h"
  18 #include "btrfs_inode.h"
  19 #include "volumes.h"
  20 #include "check-integrity.h"
  21 #include "locking.h"
  22 #include "rcu-string.h"
  23 #include "backref.h"
  24
  25 static struct kmem_cache *extent_state_cache;
  26 static struct kmem_cache *extent_buffer_cache;
  27 static struct bio_set *btrfs_bioset;
  28
  29 #ifdef CONFIG_BTRFS_DEBUG
  30 static LIST_HEAD(buffers);
  31 static LIST_HEAD(states);
  32
  33 static DEFINE_SPINLOCK(leak_lock);
  34
  35 static inline
  36 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
  37 {
  38         unsigned long flags;
  39
  40         spin_lock_irqsave(&leak_lock, flags);
  41         list_add(new, head);
  42         spin_unlock_irqrestore(&leak_lock, flags);
  43 }
  44
  45 static inline
  46 void btrfs_leak_debug_del(struct list_head *entry)
  47 {
  48         unsigned long flags;
  49
  50         spin_lock_irqsave(&leak_lock, flags);
  51         list_del(entry);
  52         spin_unlock_irqrestore(&leak_lock, flags);
  53 }
  54
  55 static inline
  56 void btrfs_leak_debug_check(void)
  57 {
  58         struct extent_state *state;
  59         struct extent_buffer *eb;
  60
  61         while (!list_empty(&states)) {
  62                 state = list_entry(states.next, struct extent_state, leak_list);
  63                 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
  64                        "state %lu in tree %p refs %d\n",
  65                        state->start, state->end, state->state, state->tree,
  66                        atomic_read(&state->refs));
  67                 list_del(&state->leak_list);
  68                 kmem_cache_free(extent_state_cache, state);
  69         }
  70
  71         while (!list_empty(&buffers)) {
  72                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  73                 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
  74                        "refs %d\n",
  75                        eb->start, eb->len, atomic_read(&eb->refs));
  76                 list_del(&eb->leak_list);
  77                 kmem_cache_free(extent_buffer_cache, eb);
  78         }
  79 }
  80
  81 #define btrfs_debug_check_extent_io_range(inode, start, end)            \
  82         __btrfs_debug_check_extent_io_range(__func__, (inode), (start), (end))
  83 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
  84                 struct inode *inode, u64 start, u64 end)
  85 {
  86         u64 isize = i_size_read(inode);
  87
  88         if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
  89                 printk_ratelimited(KERN_DEBUG
  90                     "btrfs: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
  91                                 caller, btrfs_ino(inode), isize, start, end);
  92         }
  93 }
  94 #else
  95 #define btrfs_leak_debug_add(new, head) do {} while (0)
  96 #define btrfs_leak_debug_del(entry)     do {} while (0)
  97 #define btrfs_leak_debug_check()        do {} while (0)
  98 #define btrfs_debug_check_extent_io_range(c, s, e)      do {} while (0)
  99 #endif
 100
 101 #define BUFFER_LRU_MAX 64
 102
 103 struct tree_entry {
 104         u64 start;
 105         u64 end;
 106         struct rb_node rb_node;
 107 };
 108
 109 struct extent_page_data {
 110         struct bio *bio;
 111         struct extent_io_tree *tree;
 112         get_extent_t *get_extent;
 113         unsigned long bio_flags;
 114
 115         /* tells writepage not to lock the state bits for this range
 116          * it still does the unlocking
 117          */
 118         unsigned int extent_locked:1;
 119
 120         /* tells the submit_bio code to use a WRITE_SYNC */
 121         unsigned int sync_io:1;
 122 };
 123
 124 static noinline void flush_write_bio(void *data);
 125 static inline struct btrfs_fs_info *
 126 tree_fs_info(struct extent_io_tree *tree)
 127 {
 128         return btrfs_sb(tree->mapping->host->i_sb);
 129 }
 130
 131 int __init extent_io_init(void)
 132 {
 133         extent_state_cache = kmem_cache_create("btrfs_extent_state",
 134                         sizeof(struct extent_state), 0,
 135                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
 136         if (!extent_state_cache)
 137                 return -ENOMEM;
 138
 139         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
 140                         sizeof(struct extent_buffer), 0,
 141                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
 142         if (!extent_buffer_cache)
 143                 goto free_state_cache;
 144
 145         btrfs_bioset = bioset_create(BIO_POOL_SIZE,
 146                                      offsetof(struct btrfs_io_bio, bio));
 147         if (!btrfs_bioset)
 148                 goto free_buffer_cache;
 149
 150         if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
 151                 goto free_bioset;
 152
 153         return 0;
 154
 155 free_bioset:
 156         bioset_free(btrfs_bioset);
 157         btrfs_bioset = NULL;
 158
 159 free_buffer_cache:
 160         kmem_cache_destroy(extent_buffer_cache);
 161         extent_buffer_cache = NULL;
 162
 163 free_state_cache:
 164         kmem_cache_destroy(extent_state_cache);
 165         extent_state_cache = NULL;
 166         return -ENOMEM;
 167 }
 168
 169 void extent_io_exit(void)
 170 {
 171         btrfs_leak_debug_check();
 172
 173         /*
 174          * Make sure all delayed rcu free are flushed before we
 175          * destroy caches.
 176          */
 177         rcu_barrier();
 178         if (extent_state_cache)
 179                 kmem_cache_destroy(extent_state_cache);
 180         if (extent_buffer_cache)
 181                 kmem_cache_destroy(extent_buffer_cache);
 182         if (btrfs_bioset)
 183                 bioset_free(btrfs_bioset);
 184 }
 185
 186 void extent_io_tree_init(struct extent_io_tree *tree,
 187                          struct address_space *mapping)
 188 {
 189         tree->state = RB_ROOT;
 190         INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
 191         tree->ops = NULL;
 192         tree->dirty_bytes = 0;
 193         spin_lock_init(&tree->lock);
 194         spin_lock_init(&tree->buffer_lock);
 195         tree->mapping = mapping;
 196 }
 197
 198 static struct extent_state *alloc_extent_state(gfp_t mask)
 199 {
 200         struct extent_state *state;
 201
 202         state = kmem_cache_alloc(extent_state_cache, mask);
 203         if (!state)
 204                 return state;
 205         state->state = 0;
 206         state->private = 0;
 207         state->tree = NULL;
 208         btrfs_leak_debug_add(&state->leak_list, &states);
 209         atomic_set(&state->refs, 1);
 210         init_waitqueue_head(&state->wq);
 211         trace_alloc_extent_state(state, mask, _RET_IP_);
 212         return state;
 213 }
 214
 215 void free_extent_state(struct extent_state *state)
 216 {
 217         if (!state)
 218                 return;
 219         if (atomic_dec_and_test(&state->refs)) {
 220                 WARN_ON(state->tree);
 221                 btrfs_leak_debug_del(&state->leak_list);
 222                 trace_free_extent_state(state, _RET_IP_);
 223                 kmem_cache_free(extent_state_cache, state);
 224         }
 225 }
 226
 227 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
 228                                    struct rb_node *node)
 229 {
 230         struct rb_node **p = &root->rb_node;
 231         struct rb_node *parent = NULL;
 232         struct tree_entry *entry;
 233
 234         while (*p) {
 235                 parent = *p;
 236                 entry = rb_entry(parent, struct tree_entry, rb_node);
 237
 238                 if (offset < entry->start)
 239                         p = &(*p)->rb_left;
 240                 else if (offset > entry->end)
 241                         p = &(*p)->rb_right;
 242                 else
 243                         return parent;
 244         }
 245
 246         rb_link_node(node, parent, p);
 247         rb_insert_color(node, root);
 248         return NULL;
 249 }
 250
 251 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
 252                                      struct rb_node **prev_ret,
 253                                      struct rb_node **next_ret)
 254 {
 255         struct rb_root *root = &tree->state;
 256         struct rb_node *n = root->rb_node;
 257         struct rb_node *prev = NULL;
 258         struct rb_node *orig_prev = NULL;
 259         struct tree_entry *entry;
 260         struct tree_entry *prev_entry = NULL;
 261
 262         while (n) {
 263                 entry = rb_entry(n, struct tree_entry, rb_node);
 264                 prev = n;
 265                 prev_entry = entry;
 266
 267                 if (offset < entry->start)
 268                         n = n->rb_left;
 269                 else if (offset > entry->end)
 270                         n = n->rb_right;
 271                 else
 272                         return n;
 273         }
 274
 275         if (prev_ret) {
 276                 orig_prev = prev;
 277                 while (prev && offset > prev_entry->end) {
 278                         prev = rb_next(prev);
 279                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 280                 }
 281                 *prev_ret = prev;
 282                 prev = orig_prev;
 283         }
 284
 285         if (next_ret) {
 286                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 287                 while (prev && offset < prev_entry->start) {
 288                         prev = rb_prev(prev);
 289                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 290                 }
 291                 *next_ret = prev;
 292         }
 293         return NULL;
 294 }
 295
 296 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
 297                                           u64 offset)
 298 {
 299         struct rb_node *prev = NULL;
 300         struct rb_node *ret;
 301
 302         ret = __etree_search(tree, offset, &prev, NULL);
 303         if (!ret)
 304                 return prev;
 305         return ret;
 306 }
 307
 308 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
 309                      struct extent_state *other)
 310 {
 311         if (tree->ops && tree->ops->merge_extent_hook)
 312                 tree->ops->merge_extent_hook(tree->mapping->host, new,
 313                                              other);
 314 }
 315
 316 /*
 317  * utility function to look for merge candidates inside a given range.
 318  * Any extents with matching state are merged together into a single
 319  * extent in the tree.  Extents with EXTENT_IO in their state field
 320  * are not merged because the end_io handlers need to be able to do
 321  * operations on them without sleeping (or doing allocations/splits).
 322  *
 323  * This should be called with the tree lock held.
 324  */
 325 static void merge_state(struct extent_io_tree *tree,
 326                         struct extent_state *state)
 327 {
 328         struct extent_state *other;
 329         struct rb_node *other_node;
 330
 331         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
 332                 return;
 333
 334         other_node = rb_prev(&state->rb_node);
 335         if (other_node) {
 336                 other = rb_entry(other_node, struct extent_state, rb_node);
 337                 if (other->end == state->start - 1 &&
 338                     other->state == state->state) {
 339                         merge_cb(tree, state, other);
 340                         state->start = other->start;
 341                         other->tree = NULL;
 342                         rb_erase(&other->rb_node, &tree->state);
 343                         free_extent_state(other);
 344                 }
 345         }
 346         other_node = rb_next(&state->rb_node);
 347         if (other_node) {
 348                 other = rb_entry(other_node, struct extent_state, rb_node);
 349                 if (other->start == state->end + 1 &&
 350                     other->state == state->state) {
 351                         merge_cb(tree, state, other);
 352                         state->end = other->end;
 353                         other->tree = NULL;
 354                         rb_erase(&other->rb_node, &tree->state);
 355                         free_extent_state(other);
 356                 }
 357         }
 358 }
 359
 360 static void set_state_cb(struct extent_io_tree *tree,
 361                          struct extent_state *state, unsigned long *bits)
 362 {
 363         if (tree->ops && tree->ops->set_bit_hook)
 364                 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
 365 }
 366
 367 static void clear_state_cb(struct extent_io_tree *tree,
 368                            struct extent_state *state, unsigned long *bits)
 369 {
 370         if (tree->ops && tree->ops->clear_bit_hook)
 371                 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
 372 }
 373
 374 static void set_state_bits(struct extent_io_tree *tree,
 375                            struct extent_state *state, unsigned long *bits);
 376
 377 /*
 378  * insert an extent_state struct into the tree.  'bits' are set on the
 379  * struct before it is inserted.
 380  *
 381  * This may return -EEXIST if the extent is already there, in which case the
 382  * state struct is freed.
 383  *
 384  * The tree lock is not taken internally.  This is a utility function and
 385  * probably isn't what you want to call (see set/clear_extent_bit).
 386  */
 387 static int insert_state(struct extent_io_tree *tree,
 388                         struct extent_state *state, u64 start, u64 end,
 389                         unsigned long *bits)
 390 {
 391         struct rb_node *node;
 392
 393         if (end < start)
 394                 WARN(1, KERN_ERR "btrfs end < start %llu %llu\n",
 395                        end, start);
 396         state->start = start;
 397         state->end = end;
 398
 399         set_state_bits(tree, state, bits);
 400
 401         node = tree_insert(&tree->state, end, &state->rb_node);
 402         if (node) {
 403                 struct extent_state *found;
 404                 found = rb_entry(node, struct extent_state, rb_node);
 405                 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
 406                        "%llu %llu\n",
 407                        found->start, found->end, start, end);
 408                 return -EEXIST;
 409         }
 410         state->tree = tree;
 411         merge_state(tree, state);
 412         return 0;
 413 }
 414
 415 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
 416                      u64 split)
 417 {
 418         if (tree->ops && tree->ops->split_extent_hook)
 419                 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
 420 }
 421
 422 /*
 423  * split a given extent state struct in two, inserting the preallocated
 424  * struct 'prealloc' as the newly created second half.  'split' indicates an
 425  * offset inside 'orig' where it should be split.
 426  *
 427  * Before calling,
 428  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
 429  * are two extent state structs in the tree:
 430  * prealloc: [orig->start, split - 1]
 431  * orig: [ split, orig->end ]
 432  *
 433  * The tree locks are not taken by this function. They need to be held
 434  * by the caller.
 435  */
 436 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
 437                        struct extent_state *prealloc, u64 split)
 438 {
 439         struct rb_node *node;
 440
 441         split_cb(tree, orig, split);
 442
 443         prealloc->start = orig->start;
 444         prealloc->end = split - 1;
 445         prealloc->state = orig->state;
 446         orig->start = split;
 447
 448         node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
 449         if (node) {
 450                 free_extent_state(prealloc);
 451                 return -EEXIST;
 452         }
 453         prealloc->tree = tree;
 454         return 0;
 455 }
 456
 457 static struct extent_state *next_state(struct extent_state *state)
 458 {
 459         struct rb_node *next = rb_next(&state->rb_node);
 460         if (next)
 461                 return rb_entry(next, struct extent_state, rb_node);
 462         else
 463                 return NULL;
 464 }
 465
 466 /*
 467  * utility function to clear some bits in an extent state struct.
 468  * it will optionally wake up any one waiting on this state (wake == 1).
 469  *
 470  * If no bits are set on the state struct after clearing things, the
 471  * struct is freed and removed from the tree
 472  */
 473 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
 474                                             struct extent_state *state,
 475                                             unsigned long *bits, int wake)
 476 {
 477         struct extent_state *next;
 478         unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
 479
 480         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
 481                 u64 range = state->end - state->start + 1;
 482                 WARN_ON(range > tree->dirty_bytes);
 483                 tree->dirty_bytes -= range;
 484         }
 485         clear_state_cb(tree, state, bits);
 486         state->state &= ~bits_to_clear;
 487         if (wake)
 488                 wake_up(&state->wq);
 489         if (state->state == 0) {
 490                 next = next_state(state);
 491                 if (state->tree) {
 492                         rb_erase(&state->rb_node, &tree->state);
 493                         state->tree = NULL;
 494                         free_extent_state(state);
 495                 } else {
 496                         WARN_ON(1);
 497                 }
 498         } else {
 499                 merge_state(tree, state);
 500                 next = next_state(state);
 501         }
 502         return next;
 503 }
 504
 505 static struct extent_state *
 506 alloc_extent_state_atomic(struct extent_state *prealloc)
 507 {
 508         if (!prealloc)
 509                 prealloc = alloc_extent_state(GFP_ATOMIC);
 510
 511         return prealloc;
 512 }
 513
 514 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
 515 {
 516         btrfs_panic(tree_fs_info(tree), err, "Locking error: "
 517                     "Extent tree was modified by another "
 518                     "thread while locked.");
 519 }
 520
 521 /*
 522  * clear some bits on a range in the tree.  This may require splitting
 523  * or inserting elements in the tree, so the gfp mask is used to
 524  * indicate which allocations or sleeping are allowed.
 525  *
 526  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
 527  * the given range from the tree regardless of state (ie for truncate).
 528  *
 529  * the range [start, end] is inclusive.
 530  *
 531  * This takes the tree lock, and returns 0 on success and < 0 on error.
 532  */
 533 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 534                      unsigned long bits, int wake, int delete,
 535                      struct extent_state **cached_state,
 536                      gfp_t mask)
 537 {
 538         struct extent_state *state;
 539         struct extent_state *cached;
 540         struct extent_state *prealloc = NULL;
 541         struct rb_node *node;
 542         u64 last_end;
 543         int err;
 544         int clear = 0;
 545
 546         btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
 547
 548         if (bits & EXTENT_DELALLOC)
 549                 bits |= EXTENT_NORESERVE;
 550
 551         if (delete)
 552                 bits |= ~EXTENT_CTLBITS;
 553         bits |= EXTENT_FIRST_DELALLOC;
 554
 555         if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
 556                 clear = 1;
 557 again:
 558         if (!prealloc && (mask & __GFP_WAIT)) {
 559                 prealloc = alloc_extent_state(mask);
 560                 if (!prealloc)
 561                         return -ENOMEM;
 562         }
 563
 564         spin_lock(&tree->lock);
 565         if (cached_state) {
 566                 cached = *cached_state;
 567
 568                 if (clear) {
 569                         *cached_state = NULL;
 570                         cached_state = NULL;
 571                 }
 572
 573                 if (cached && cached->tree && cached->start <= start &&
 574                     cached->end > start) {
 575                         if (clear)
 576                                 atomic_dec(&cached->refs);
 577                         state = cached;
 578                         goto hit_next;
 579                 }
 580                 if (clear)
 581                         free_extent_state(cached);
 582         }
 583         /*
 584          * this search will find the extents that end after
 585          * our range starts
 586          */
 587         node = tree_search(tree, start);
 588         if (!node)
 589                 goto out;
 590         state = rb_entry(node, struct extent_state, rb_node);
 591 hit_next:
 592         if (state->start > end)
 593                 goto out;
 594         WARN_ON(state->end < start);
 595         last_end = state->end;
 596
 597         /* the state doesn't have the wanted bits, go ahead */
 598         if (!(state->state & bits)) {
 599                 state = next_state(state);
 600                 goto next;
 601         }
 602
 603         /*
 604          *     | ---- desired range ---- |
 605          *  | state | or
 606          *  | ------------- state -------------- |
 607          *
 608          * We need to split the extent we found, and may flip
 609          * bits on second half.
 610          *
 611          * If the extent we found extends past our range, we
 612          * just split and search again.  It'll get split again
 613          * the next time though.
 614          *
 615          * If the extent we found is inside our range, we clear
 616          * the desired bit on it.
 617          */
 618
 619         if (state->start < start) {
 620                 prealloc = alloc_extent_state_atomic(prealloc);
 621                 BUG_ON(!prealloc);
 622                 err = split_state(tree, state, prealloc, start);
 623                 if (err)
 624                         extent_io_tree_panic(tree, err);
 625
 626                 prealloc = NULL;
 627                 if (err)
 628                         goto out;
 629                 if (state->end <= end) {
 630                         state = clear_state_bit(tree, state, &bits, wake);
 631                         goto next;
 632                 }
 633                 goto search_again;
 634         }
 635         /*
 636          * | ---- desired range ---- |
 637          *                        | state |
 638          * We need to split the extent, and clear the bit
 639          * on the first half
 640          */
 641         if (state->start <= end && state->end > end) {
 642                 prealloc = alloc_extent_state_atomic(prealloc);
 643                 BUG_ON(!prealloc);
 644                 err = split_state(tree, state, prealloc, end + 1);
 645                 if (err)
 646                         extent_io_tree_panic(tree, err);
 647
 648                 if (wake)
 649                         wake_up(&state->wq);
 650
 651                 clear_state_bit(tree, prealloc, &bits, wake);
 652
 653                 prealloc = NULL;
 654                 goto out;
 655         }
 656
 657         state = clear_state_bit(tree, state, &bits, wake);
 658 next:
 659         if (last_end == (u64)-1)
 660                 goto out;
 661         start = last_end + 1;
 662         if (start <= end && state && !need_resched())
 663                 goto hit_next;
 664         goto search_again;
 665
 666 out:
 667         spin_unlock(&tree->lock);
 668         if (prealloc)
 669                 free_extent_state(prealloc);
 670
 671         return 0;
 672
 673 search_again:
 674         if (start > end)
 675                 goto out;
 676         spin_unlock(&tree->lock);
 677         if (mask & __GFP_WAIT)
 678                 cond_resched();
 679         goto again;
 680 }
 681
 682 static void wait_on_state(struct extent_io_tree *tree,
 683                           struct extent_state *state)
 684                 __releases(tree->lock)
 685                 __acquires(tree->lock)
 686 {
 687         DEFINE_WAIT(wait);
 688         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
 689         spin_unlock(&tree->lock);
 690         schedule();
 691         spin_lock(&tree->lock);
 692         finish_wait(&state->wq, &wait);
 693 }
 694
 695 /*
 696  * waits for one or more bits to clear on a range in the state tree.
 697  * The range [start, end] is inclusive.
 698  * The tree lock is taken by this function
 699  */
 700 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 701                             unsigned long bits)
 702 {
 703         struct extent_state *state;
 704         struct rb_node *node;
 705
 706         btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
 707
 708         spin_lock(&tree->lock);
 709 again:
 710         while (1) {
 711                 /*
 712                  * this search will find all the extents that end after
 713                  * our range starts
 714                  */
 715                 node = tree_search(tree, start);
 716                 if (!node)
 717                         break;
 718
 719                 state = rb_entry(node, struct extent_state, rb_node);
 720
 721                 if (state->start > end)
 722                         goto out;
 723
 724                 if (state->state & bits) {
 725                         start = state->start;
 726                         atomic_inc(&state->refs);
 727                         wait_on_state(tree, state);
 728                         free_extent_state(state);
 729                         goto again;
 730                 }
 731                 start = state->end + 1;
 732
 733                 if (start > end)
 734                         break;
 735
 736                 cond_resched_lock(&tree->lock);
 737         }
 738 out:
 739         spin_unlock(&tree->lock);
 740 }
 741
 742 static void set_state_bits(struct extent_io_tree *tree,
 743                            struct extent_state *state,
 744                            unsigned long *bits)
 745 {
 746         unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
 747
 748         set_state_cb(tree, state, bits);
 749         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
 750                 u64 range = state->end - state->start + 1;
 751                 tree->dirty_bytes += range;
 752         }
 753         state->state |= bits_to_set;
 754 }
 755
 756 static void cache_state(struct extent_state *state,
 757                         struct extent_state **cached_ptr)
 758 {
 759         if (cached_ptr && !(*cached_ptr)) {
 760                 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
 761                         *cached_ptr = state;
 762                         atomic_inc(&state->refs);
 763                 }
 764         }
 765 }
 766
 767 /*
 768  * set some bits on a range in the tree.  This may require allocations or
 769  * sleeping, so the gfp mask is used to indicate what is allowed.
 770  *
 771  * If any of the exclusive bits are set, this will fail with -EEXIST if some
 772  * part of the range already has the desired bits set.  The start of the
 773  * existing range is returned in failed_start in this case.
 774  *
 775  * [start, end] is inclusive This takes the tree lock.
 776  */
 777
 778 static int __must_check
 779 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 780                  unsigned long bits, unsigned long exclusive_bits,
 781                  u64 *failed_start, struct extent_state **cached_state,
 782                  gfp_t mask)
 783 {
 784         struct extent_state *state;
 785         struct extent_state *prealloc = NULL;
 786         struct rb_node *node;
 787         int err = 0;
 788         u64 last_start;
 789         u64 last_end;
 790
 791         btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
 792
 793         bits |= EXTENT_FIRST_DELALLOC;
 794 again:
 795         if (!prealloc && (mask & __GFP_WAIT)) {
 796                 prealloc = alloc_extent_state(mask);
 797                 BUG_ON(!prealloc);
 798         }
 799
 800         spin_lock(&tree->lock);
 801         if (cached_state && *cached_state) {
 802                 state = *cached_state;
 803                 if (state->start <= start && state->end > start &&
 804                     state->tree) {
 805                         node = &state->rb_node;
 806                         goto hit_next;
 807                 }
 808         }
 809         /*
 810          * this search will find all the extents that end after
 811          * our range starts.
 812          */
 813         node = tree_search(tree, start);
 814         if (!node) {
 815                 prealloc = alloc_extent_state_atomic(prealloc);
 816                 BUG_ON(!prealloc);
 817                 err = insert_state(tree, prealloc, start, end, &bits);
 818                 if (err)
 819                         extent_io_tree_panic(tree, err);
 820
 821                 prealloc = NULL;
 822                 goto out;
 823         }
 824         state = rb_entry(node, struct extent_state, rb_node);
 825 hit_next:
 826         last_start = state->start;
 827         last_end = state->end;
 828
 829         /*
 830          * | ---- desired range ---- |
 831          * | state |
 832          *
 833          * Just lock what we found and keep going
 834          */
 835         if (state->start == start && state->end <= end) {
 836                 if (state->state & exclusive_bits) {
 837                         *failed_start = state->start;
 838                         err = -EEXIST;
 839                         goto out;
 840                 }
 841
 842                 set_state_bits(tree, state, &bits);
 843                 cache_state(state, cached_state);
 844                 merge_state(tree, state);
 845                 if (last_end == (u64)-1)
 846                         goto out;
 847                 start = last_end + 1;
 848                 state = next_state(state);
 849                 if (start < end && state && state->start == start &&
 850                     !need_resched())
 851                         goto hit_next;
 852                 goto search_again;
 853         }
 854
 855         /*
 856          *     | ---- desired range ---- |
 857          * | state |
 858          *   or
 859          * | ------------- state -------------- |
 860          *
 861          * We need to split the extent we found, and may flip bits on
 862          * second half.
 863          *
 864          * If the extent we found extends past our
 865          * range, we just split and search again.  It'll get split
 866          * again the next time though.
 867          *
 868          * If the extent we found is inside our range, we set the
 869          * desired bit on it.
 870          */
 871         if (state->start < start) {
 872                 if (state->state & exclusive_bits) {
 873                         *failed_start = start;
 874                         err = -EEXIST;
 875                         goto out;
 876                 }
 877
 878                 prealloc = alloc_extent_state_atomic(prealloc);
 879                 BUG_ON(!prealloc);
 880                 err = split_state(tree, state, prealloc, start);
 881                 if (err)
 882                         extent_io_tree_panic(tree, err);
 883
 884                 prealloc = NULL;
 885                 if (err)
 886                         goto out;
 887                 if (state->end <= end) {
 888                         set_state_bits(tree, state, &bits);
 889                         cache_state(state, cached_state);
 890                         merge_state(tree, state);
 891                         if (last_end == (u64)-1)
 892                                 goto out;
 893                         start = last_end + 1;
 894                         state = next_state(state);
 895                         if (start < end && state && state->start == start &&
 896                             !need_resched())
 897                                 goto hit_next;
 898                 }
 899                 goto search_again;
 900         }
 901         /*
 902          * | ---- desired range ---- |
 903          *     | state | or               | state |
 904          *
 905          * There's a hole, we need to insert something in it and
 906          * ignore the extent we found.
 907          */
 908         if (state->start > start) {
 909                 u64 this_end;
 910                 if (end < last_start)
 911                         this_end = end;
 912                 else
 913                         this_end = last_start - 1;
 914
 915                 prealloc = alloc_extent_state_atomic(prealloc);
 916                 BUG_ON(!prealloc);
 917
 918                 /*
 919                  * Avoid to free 'prealloc' if it can be merged with
 920                  * the later extent.
 921                  */
 922                 err = insert_state(tree, prealloc, start, this_end,
 923                                    &bits);
 924                 if (err)
 925                         extent_io_tree_panic(tree, err);
 926
 927                 cache_state(prealloc, cached_state);
 928                 prealloc = NULL;
 929                 start = this_end + 1;
 930                 goto search_again;
 931         }
 932         /*
 933          * | ---- desired range ---- |
 934          *                        | state |
 935          * We need to split the extent, and set the bit
 936          * on the first half
 937          */
 938         if (state->start <= end && state->end > end) {
 939                 if (state->state & exclusive_bits) {
 940                         *failed_start = start;
 941                         err = -EEXIST;
 942                         goto out;
 943                 }
 944
 945                 prealloc = alloc_extent_state_atomic(prealloc);
 946                 BUG_ON(!prealloc);
 947                 err = split_state(tree, state, prealloc, end + 1);
 948                 if (err)
 949                         extent_io_tree_panic(tree, err);
 950
 951                 set_state_bits(tree, prealloc, &bits);
 952                 cache_state(prealloc, cached_state);
 953                 merge_state(tree, prealloc);
 954                 prealloc = NULL;
 955                 goto out;
 956         }
 957
 958         goto search_again;
 959
 960 out:
 961         spin_unlock(&tree->lock);
 962         if (prealloc)
 963                 free_extent_state(prealloc);
 964
 965         return err;
 966
 967 search_again:
 968         if (start > end)
 969                 goto out;
 970         spin_unlock(&tree->lock);
 971         if (mask & __GFP_WAIT)
 972                 cond_resched();
 973         goto again;
 974 }
 975
 976 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 977                    unsigned long bits, u64 * failed_start,
 978                    struct extent_state **cached_state, gfp_t mask)
 979 {
 980         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
 981                                 cached_state, mask);
 982 }
 983
 984
 985 /**
 986  * convert_extent_bit - convert all bits in a given range from one bit to
 987  *                      another
 988  * @tree:       the io tree to search
 989  * @start:      the start offset in bytes
 990  * @end:        the end offset in bytes (inclusive)
 991  * @bits:       the bits to set in this range
 992  * @clear_bits: the bits to clear in this range
 993  * @cached_state:       state that we're going to cache
 994  * @mask:       the allocation mask
 995  *
 996  * This will go through and set bits for the given range.  If any states exist
 997  * already in this range they are set with the given bit and cleared of the
 998  * clear_bits.  This is only meant to be used by things that are mergeable, ie
 999  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1000  * boundary bits like LOCK.
1001  */
1002 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1003                        unsigned long bits, unsigned long clear_bits,
1004                        struct extent_state **cached_state, gfp_t mask)
1005 {
1006         struct extent_state *state;
1007         struct extent_state *prealloc = NULL;
1008         struct rb_node *node;
1009         int err = 0;
1010         u64 last_start;
1011         u64 last_end;
1012
1013         btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
1014
1015 again:
1016         if (!prealloc && (mask & __GFP_WAIT)) {
1017                 prealloc = alloc_extent_state(mask);
1018                 if (!prealloc)
1019                         return -ENOMEM;
1020         }
1021
1022         spin_lock(&tree->lock);
1023         if (cached_state && *cached_state) {
1024                 state = *cached_state;
1025                 if (state->start <= start && state->end > start &&
1026                     state->tree) {
1027                         node = &state->rb_node;
1028                         goto hit_next;
1029                 }
1030         }
1031
1032         /*
1033          * this search will find all the extents that end after
1034          * our range starts.
1035          */
1036         node = tree_search(tree, start);
1037         if (!node) {
1038                 prealloc = alloc_extent_state_atomic(prealloc);
1039                 if (!prealloc) {
1040                         err = -ENOMEM;
1041                         goto out;
1042                 }
1043                 err = insert_state(tree, prealloc, start, end, &bits);
1044                 prealloc = NULL;
1045                 if (err)
1046                         extent_io_tree_panic(tree, err);
1047                 goto out;
1048         }
1049         state = rb_entry(node, struct extent_state, rb_node);
1050 hit_next:
1051         last_start = state->start;
1052         last_end = state->end;
1053
1054         /*
1055          * | ---- desired range ---- |
1056          * | state |
1057          *
1058          * Just lock what we found and keep going
1059          */
1060         if (state->start == start && state->end <= end) {
1061                 set_state_bits(tree, state, &bits);
1062                 cache_state(state, cached_state);
1063                 state = clear_state_bit(tree, state, &clear_bits, 0);
1064                 if (last_end == (u64)-1)
1065                         goto out;
1066                 start = last_end + 1;
1067                 if (start < end && state && state->start == start &&
1068                     !need_resched())
1069                         goto hit_next;
1070                 goto search_again;
1071         }
1072
1073         /*
1074          *     | ---- desired range ---- |
1075          * | state |
1076          *   or
1077          * | ------------- state -------------- |
1078          *
1079          * We need to split the extent we found, and may flip bits on
1080          * second half.
1081          *
1082          * If the extent we found extends past our
1083          * range, we just split and search again.  It'll get split
1084          * again the next time though.
1085          *
1086          * If the extent we found is inside our range, we set the
1087          * desired bit on it.
1088          */
1089         if (state->start < start) {
1090                 prealloc = alloc_extent_state_atomic(prealloc);
1091                 if (!prealloc) {
1092                         err = -ENOMEM;
1093                         goto out;
1094                 }
1095                 err = split_state(tree, state, prealloc, start);
1096                 if (err)
1097                         extent_io_tree_panic(tree, err);
1098                 prealloc = NULL;
1099                 if (err)
1100                         goto out;
1101                 if (state->end <= end) {
1102                         set_state_bits(tree, state, &bits);
1103                         cache_state(state, cached_state);
1104                         state = clear_state_bit(tree, state, &clear_bits, 0);
1105                         if (last_end == (u64)-1)
1106                                 goto out;
1107                         start = last_end + 1;
1108                         if (start < end && state && state->start == start &&
1109                             !need_resched())
1110                                 goto hit_next;
1111                 }
1112                 goto search_again;
1113         }
1114         /*
1115          * | ---- desired range ---- |
1116          *     | state | or               | state |
1117          *
1118          * There's a hole, we need to insert something in it and
1119          * ignore the extent we found.
1120          */
1121         if (state->start > start) {
1122                 u64 this_end;
1123                 if (end < last_start)
1124                         this_end = end;
1125                 else
1126                         this_end = last_start - 1;
1127
1128                 prealloc = alloc_extent_state_atomic(prealloc);
1129                 if (!prealloc) {
1130                         err = -ENOMEM;
1131                         goto out;
1132                 }
1133
1134                 /*
1135                  * Avoid to free 'prealloc' if it can be merged with
1136                  * the later extent.
1137                  */
1138                 err = insert_state(tree, prealloc, start, this_end,
1139                                    &bits);
1140                 if (err)
1141                         extent_io_tree_panic(tree, err);
1142                 cache_state(prealloc, cached_state);
1143                 prealloc = NULL;
1144                 start = this_end + 1;
1145                 goto search_again;
1146         }
1147         /*
1148          * | ---- desired range ---- |
1149          *                        | state |
1150          * We need to split the extent, and set the bit
1151          * on the first half
1152          */
1153         if (state->start <= end && state->end > end) {
1154                 prealloc = alloc_extent_state_atomic(prealloc);
1155                 if (!prealloc) {
1156                         err = -ENOMEM;
1157                         goto out;
1158                 }
1159
1160                 err = split_state(tree, state, prealloc, end + 1);
1161                 if (err)
1162                         extent_io_tree_panic(tree, err);
1163
1164                 set_state_bits(tree, prealloc, &bits);
1165                 cache_state(prealloc, cached_state);
1166                 clear_state_bit(tree, prealloc, &clear_bits, 0);
1167                 prealloc = NULL;
1168                 goto out;
1169         }
1170
1171         goto search_again;
1172
1173 out:
1174         spin_unlock(&tree->lock);
1175         if (prealloc)
1176                 free_extent_state(prealloc);
1177
1178         return err;
1179
1180 search_again:
1181         if (start > end)
1182                 goto out;
1183         spin_unlock(&tree->lock);
1184         if (mask & __GFP_WAIT)
1185                 cond_resched();
1186         goto again;
1187 }
1188
1189 /* wrappers around set/clear extent bit */
1190 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1191                      gfp_t mask)
1192 {
1193         return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
1194                               NULL, mask);
1195 }
1196
1197 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1198                     unsigned long bits, gfp_t mask)
1199 {
1200         return set_extent_bit(tree, start, end, bits, NULL,
1201                               NULL, mask);
1202 }
1203
1204 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1205                       unsigned long bits, gfp_t mask)
1206 {
1207         return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
1208 }
1209
1210 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
1211                         struct extent_state **cached_state, gfp_t mask)
1212 {
1213         return set_extent_bit(tree, start, end,
1214                               EXTENT_DELALLOC | EXTENT_UPTODATE,
1215                               NULL, cached_state, mask);
1216 }
1217
1218 int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1219                       struct extent_state **cached_state, gfp_t mask)
1220 {
1221         return set_extent_bit(tree, start, end,
1222                               EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1223                               NULL, cached_state, mask);
1224 }
1225
1226 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1227                        gfp_t mask)
1228 {
1229         return clear_extent_bit(tree, start, end,
1230                                 EXTENT_DIRTY | EXTENT_DELALLOC |
1231                                 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
1232 }
1233
1234 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1235                      gfp_t mask)
1236 {
1237         return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
1238                               NULL, mask);
1239 }
1240
1241 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1242                         struct extent_state **cached_state, gfp_t mask)
1243 {
1244         return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
1245                               cached_state, mask);
1246 }
1247
1248 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1249                           struct extent_state **cached_state, gfp_t mask)
1250 {
1251         return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
1252                                 cached_state, mask);
1253 }
1254
1255 /*
1256  * either insert or lock state struct between start and end use mask to tell
1257  * us if waiting is desired.
1258  */
1259 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1260                      unsigned long bits, struct extent_state **cached_state)
1261 {
1262         int err;
1263         u64 failed_start;
1264         while (1) {
1265                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1266                                        EXTENT_LOCKED, &failed_start,
1267                                        cached_state, GFP_NOFS);
1268                 if (err == -EEXIST) {
1269                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1270                         start = failed_start;
1271                 } else
1272                         break;
1273                 WARN_ON(start > end);
1274         }
1275         return err;
1276 }
1277
1278 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1279 {
1280         return lock_extent_bits(tree, start, end, 0, NULL);
1281 }
1282
1283 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1284 {
1285         int err;
1286         u64 failed_start;
1287
1288         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1289                                &failed_start, NULL, GFP_NOFS);
1290         if (err == -EEXIST) {
1291                 if (failed_start > start)
1292                         clear_extent_bit(tree, start, failed_start - 1,
1293                                          EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
1294                 return 0;
1295         }
1296         return 1;
1297 }
1298
1299 int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1300                          struct extent_state **cached, gfp_t mask)
1301 {
1302         return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1303                                 mask);
1304 }
1305
1306 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1307 {
1308         return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1309                                 GFP_NOFS);
1310 }
1311
1312 int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1313 {
1314         unsigned long index = start >> PAGE_CACHE_SHIFT;
1315         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1316         struct page *page;
1317
1318         while (index <= end_index) {
1319                 page = find_get_page(inode->i_mapping, index);
1320                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1321                 clear_page_dirty_for_io(page);
1322                 page_cache_release(page);
1323                 index++;
1324         }
1325         return 0;
1326 }
1327
1328 int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1329 {
1330         unsigned long index = start >> PAGE_CACHE_SHIFT;
1331         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1332         struct page *page;
1333
1334         while (index <= end_index) {
1335                 page = find_get_page(inode->i_mapping, index);
1336                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1337                 account_page_redirty(page);
1338                 __set_page_dirty_nobuffers(page);
1339                 page_cache_release(page);
1340                 index++;
1341         }
1342         return 0;
1343 }
1344
1345 /*
1346  * helper function to set both pages and extents in the tree writeback
1347  */
1348 static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1349 {
1350         unsigned long index = start >> PAGE_CACHE_SHIFT;
1351         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1352         struct page *page;
1353
1354         while (index <= end_index) {
1355                 page = find_get_page(tree->mapping, index);
1356                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1357                 set_page_writeback(page);
1358                 page_cache_release(page);
1359                 index++;
1360         }
1361         return 0;
1362 }
1363
1364 /* find the first state struct with 'bits' set after 'start', and
1365  * return it.  tree->lock must be held.  NULL will returned if
1366  * nothing was found after 'start'
1367  */
1368 static struct extent_state *
1369 find_first_extent_bit_state(struct extent_io_tree *tree,
1370                             u64 start, unsigned long bits)
1371 {
1372         struct rb_node *node;
1373         struct extent_state *state;
1374
1375         /*
1376          * this search will find all the extents that end after
1377          * our range starts.
1378          */
1379         node = tree_search(tree, start);
1380         if (!node)
1381                 goto out;
1382
1383         while (1) {
1384                 state = rb_entry(node, struct extent_state, rb_node);
1385                 if (state->end >= start && (state->state & bits))
1386                         return state;
1387
1388                 node = rb_next(node);
1389                 if (!node)
1390                         break;
1391         }
1392 out:
1393         return NULL;
1394 }
1395
1396 /*
1397  * find the first offset in the io tree with 'bits' set. zero is
1398  * returned if we find something, and *start_ret and *end_ret are
1399  * set to reflect the state struct that was found.
1400  *
1401  * If nothing was found, 1 is returned. If found something, return 0.
1402  */
1403 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1404                           u64 *start_ret, u64 *end_ret, unsigned long bits,
1405                           struct extent_state **cached_state)
1406 {
1407         struct extent_state *state;
1408         struct rb_node *n;
1409         int ret = 1;
1410
1411         spin_lock(&tree->lock);
1412         if (cached_state && *cached_state) {
1413                 state = *cached_state;
1414                 if (state->end == start - 1 && state->tree) {
1415                         n = rb_next(&state->rb_node);
1416                         while (n) {
1417                                 state = rb_entry(n, struct extent_state,
1418                                                  rb_node);
1419                                 if (state->state & bits)
1420                                         goto got_it;
1421                                 n = rb_next(n);
1422                         }
1423                         free_extent_state(*cached_state);
1424                         *cached_state = NULL;
1425                         goto out;
1426                 }
1427                 free_extent_state(*cached_state);
1428                 *cached_state = NULL;
1429         }
1430
1431         state = find_first_extent_bit_state(tree, start, bits);
1432 got_it:
1433         if (state) {
1434                 cache_state(state, cached_state);
1435                 *start_ret = state->start;
1436                 *end_ret = state->end;
1437                 ret = 0;
1438         }
1439 out:
1440         spin_unlock(&tree->lock);
1441         return ret;
1442 }
1443
1444 /*
1445  * find a contiguous range of bytes in the file marked as delalloc, not
1446  * more than 'max_bytes'.  start and end are used to return the range,
1447  *
1448  * 1 is returned if we find something, 0 if nothing was in the tree
1449  */
1450 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1451                                         u64 *start, u64 *end, u64 max_bytes,
1452                                         struct extent_state **cached_state)
1453 {
1454         struct rb_node *node;
1455         struct extent_state *state;
1456         u64 cur_start = *start;
1457         u64 found = 0;
1458         u64 total_bytes = 0;
1459
1460         spin_lock(&tree->lock);
1461
1462         /*
1463          * this search will find all the extents that end after
1464          * our range starts.
1465          */
1466         node = tree_search(tree, cur_start);
1467         if (!node) {
1468                 if (!found)
1469                         *end = (u64)-1;
1470                 goto out;
1471         }
1472
1473         while (1) {
1474                 state = rb_entry(node, struct extent_state, rb_node);
1475                 if (found && (state->start != cur_start ||
1476                               (state->state & EXTENT_BOUNDARY))) {
1477                         goto out;
1478                 }
1479                 if (!(state->state & EXTENT_DELALLOC)) {
1480                         if (!found)
1481                                 *end = state->end;
1482                         goto out;
1483                 }
1484                 if (!found) {
1485                         *start = state->start;
1486                         *cached_state = state;
1487                         atomic_inc(&state->refs);
1488                 }
1489                 found++;
1490                 *end = state->end;
1491                 cur_start = state->end + 1;
1492                 node = rb_next(node);
1493                 total_bytes += state->end - state->start + 1;
1494                 if (total_bytes >= max_bytes)
1495                         break;
1496                 if (!node)
1497                         break;
1498         }
1499 out:
1500         spin_unlock(&tree->lock);
1501         return found;
1502 }
1503
1504 static noinline void __unlock_for_delalloc(struct inode *inode,
1505                                            struct page *locked_page,
1506                                            u64 start, u64 end)
1507 {
1508         int ret;
1509         struct page *pages[16];
1510         unsigned long index = start >> PAGE_CACHE_SHIFT;
1511         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1512         unsigned long nr_pages = end_index - index + 1;
1513         int i;
1514
1515         if (index == locked_page->index && end_index == index)
1516                 return;
1517
1518         while (nr_pages > 0) {
1519                 ret = find_get_pages_contig(inode->i_mapping, index,
1520                                      min_t(unsigned long, nr_pages,
1521                                      ARRAY_SIZE(pages)), pages);
1522                 for (i = 0; i < ret; i++) {
1523                         if (pages[i] != locked_page)
1524                                 unlock_page(pages[i]);
1525                         page_cache_release(pages[i]);
1526                 }
1527                 nr_pages -= ret;
1528                 index += ret;
1529                 cond_resched();
1530         }
1531 }
1532
1533 static noinline int lock_delalloc_pages(struct inode *inode,
1534                                         struct page *locked_page,
1535                                         u64 delalloc_start,
1536                                         u64 delalloc_end)
1537 {
1538         unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1539         unsigned long start_index = index;
1540         unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1541         unsigned long pages_locked = 0;
1542         struct page *pages[16];
1543         unsigned long nrpages;
1544         int ret;
1545         int i;
1546
1547         /* the caller is responsible for locking the start index */
1548         if (index == locked_page->index && index == end_index)
1549                 return 0;
1550
1551         /* skip the page at the start index */
1552         nrpages = end_index - index + 1;
1553         while (nrpages > 0) {
1554                 ret = find_get_pages_contig(inode->i_mapping, index,
1555                                      min_t(unsigned long,
1556                                      nrpages, ARRAY_SIZE(pages)), pages);
1557                 if (ret == 0) {
1558                         ret = -EAGAIN;
1559                         goto done;
1560                 }
1561                 /* now we have an array of pages, lock them all */
1562                 for (i = 0; i < ret; i++) {
1563                         /*
1564                          * the caller is taking responsibility for
1565                          * locked_page
1566                          */
1567                         if (pages[i] != locked_page) {
1568                                 lock_page(pages[i]);
1569                                 if (!PageDirty(pages[i]) ||
1570                                     pages[i]->mapping != inode->i_mapping) {
1571                                         ret = -EAGAIN;
1572                                         unlock_page(pages[i]);
1573                                         page_cache_release(pages[i]);
1574                                         goto done;
1575                                 }
1576                         }
1577                         page_cache_release(pages[i]);
1578                         pages_locked++;
1579                 }
1580                 nrpages -= ret;
1581                 index += ret;
1582                 cond_resched();
1583         }
1584         ret = 0;
1585 done:
1586         if (ret && pages_locked) {
1587                 __unlock_for_delalloc(inode, locked_page,
1588                               delalloc_start,
1589                               ((u64)(start_index + pages_locked - 1)) <<
1590                               PAGE_CACHE_SHIFT);
1591         }
1592         return ret;
1593 }
1594
1595 /*
1596  * find a contiguous range of bytes in the file marked as delalloc, not
1597  * more than 'max_bytes'.  start and end are used to return the range,
1598  *
1599  * 1 is returned if we find something, 0 if nothing was in the tree
1600  */
1601 static noinline u64 find_lock_delalloc_range(struct inode *inode,
1602                                              struct extent_io_tree *tree,
1603                                              struct page *locked_page,
1604                                              u64 *start, u64 *end,
1605                                              u64 max_bytes)
1606 {
1607         u64 delalloc_start;
1608         u64 delalloc_end;
1609         u64 found;
1610         struct extent_state *cached_state = NULL;
1611         int ret;
1612         int loops = 0;
1613
1614 again:
1615         /* step one, find a bunch of delalloc bytes starting at start */
1616         delalloc_start = *start;
1617         delalloc_end = 0;
1618         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1619                                     max_bytes, &cached_state);
1620         if (!found || delalloc_end <= *start) {
1621                 *start = delalloc_start;
1622                 *end = delalloc_end;
1623                 free_extent_state(cached_state);
1624                 return 0;
1625         }
1626
1627         /*
1628          * start comes from the offset of locked_page.  We have to lock
1629          * pages in order, so we can't process delalloc bytes before
1630          * locked_page
1631          */
1632         if (delalloc_start < *start)
1633                 delalloc_start = *start;
1634
1635         /*
1636          * make sure to limit the number of pages we try to lock down
1637          */
1638         if (delalloc_end + 1 - delalloc_start > max_bytes)
1639                 delalloc_end = delalloc_start + max_bytes - 1;
1640
1641         /* step two, lock all the pages after the page that has start */
1642         ret = lock_delalloc_pages(inode, locked_page,
1643                                   delalloc_start, delalloc_end);
1644         if (ret == -EAGAIN) {
1645                 /* some of the pages are gone, lets avoid looping by
1646                  * shortening the size of the delalloc range we're searching
1647                  */
1648                 free_extent_state(cached_state);
1649                 if (!loops) {
1650                         max_bytes = PAGE_CACHE_SIZE;
1651                         loops = 1;
1652                         goto again;
1653                 } else {
1654                         found = 0;
1655                         goto out_failed;
1656                 }
1657         }
1658         BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1659
1660         /* step three, lock the state bits for the whole range */
1661         lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
1662
1663         /* then test to make sure it is all still delalloc */
1664         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1665                              EXTENT_DELALLOC, 1, cached_state);
1666         if (!ret) {
1667                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1668                                      &cached_state, GFP_NOFS);
1669                 __unlock_for_delalloc(inode, locked_page,
1670                               delalloc_start, delalloc_end);
1671                 cond_resched();
1672                 goto again;
1673         }
1674         free_extent_state(cached_state);
1675         *start = delalloc_start;
1676         *end = delalloc_end;
1677 out_failed:
1678         return found;
1679 }
1680
1681 int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1682                                  struct page *locked_page,
1683                                  unsigned long clear_bits,
1684                                  unsigned long page_ops)
1685 {
1686         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1687         int ret;
1688         struct page *pages[16];
1689         unsigned long index = start >> PAGE_CACHE_SHIFT;
1690         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1691         unsigned long nr_pages = end_index - index + 1;
1692         int i;
1693
1694         clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1695         if (page_ops == 0)
1696                 return 0;
1697
1698         while (nr_pages > 0) {
1699                 ret = find_get_pages_contig(inode->i_mapping, index,
1700                                      min_t(unsigned long,
1701                                      nr_pages, ARRAY_SIZE(pages)), pages);
1702                 for (i = 0; i < ret; i++) {
1703
1704                         if (page_ops & PAGE_SET_PRIVATE2)
1705                                 SetPagePrivate2(pages[i]);
1706
1707                         if (pages[i] == locked_page) {
1708                                 page_cache_release(pages[i]);
1709                                 continue;
1710                         }
1711                         if (page_ops & PAGE_CLEAR_DIRTY)
1712                                 clear_page_dirty_for_io(pages[i]);
1713                         if (page_ops & PAGE_SET_WRITEBACK)
1714                                 set_page_writeback(pages[i]);
1715                         if (page_ops & PAGE_END_WRITEBACK)
1716                                 end_page_writeback(pages[i]);
1717                         if (page_ops & PAGE_UNLOCK)
1718                                 unlock_page(pages[i]);
1719                         page_cache_release(pages[i]);
1720                 }
1721                 nr_pages -= ret;
1722                 index += ret;
1723                 cond_resched();
1724         }
1725         return 0;
1726 }
1727
1728 /*
1729  * count the number of bytes in the tree that have a given bit(s)
1730  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1731  * cached.  The total number found is returned.
1732  */
1733 u64 count_range_bits(struct extent_io_tree *tree,
1734                      u64 *start, u64 search_end, u64 max_bytes,
1735                      unsigned long bits, int contig)
1736 {
1737         struct rb_node *node;
1738         struct extent_state *state;
1739         u64 cur_start = *start;
1740         u64 total_bytes = 0;
1741         u64 last = 0;
1742         int found = 0;
1743
1744         if (search_end <= cur_start) {
1745                 WARN_ON(1);
1746                 return 0;
1747         }
1748
1749         spin_lock(&tree->lock);
1750         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1751                 total_bytes = tree->dirty_bytes;
1752                 goto out;
1753         }
1754         /*
1755          * this search will find all the extents that end after
1756          * our range starts.
1757          */
1758         node = tree_search(tree, cur_start);
1759         if (!node)
1760                 goto out;
1761
1762         while (1) {
1763                 state = rb_entry(node, struct extent_state, rb_node);
1764                 if (state->start > search_end)
1765                         break;
1766                 if (contig && found && state->start > last + 1)
1767                         break;
1768                 if (state->end >= cur_start && (state->state & bits) == bits) {
1769                         total_bytes += min(search_end, state->end) + 1 -
1770                                        max(cur_start, state->start);
1771                         if (total_bytes >= max_bytes)
1772                                 break;
1773                         if (!found) {
1774                                 *start = max(cur_start, state->start);
1775                                 found = 1;
1776                         }
1777                         last = state->end;
1778                 } else if (contig && found) {
1779                         break;
1780                 }
1781                 node = rb_next(node);
1782                 if (!node)
1783                         break;
1784         }
1785 out:
1786         spin_unlock(&tree->lock);
1787         return total_bytes;
1788 }
1789
1790 /*
1791  * set the private field for a given byte offset in the tree.  If there isn't
1792  * an extent_state there already, this does nothing.
1793  */
1794 static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1795 {
1796         struct rb_node *node;
1797         struct extent_state *state;
1798         int ret = 0;
1799
1800         spin_lock(&tree->lock);
1801         /*
1802          * this search will find all the extents that end after
1803          * our range starts.
1804          */
1805         node = tree_search(tree, start);
1806         if (!node) {
1807                 ret = -ENOENT;
1808                 goto out;
1809         }
1810         state = rb_entry(node, struct extent_state, rb_node);
1811         if (state->start != start) {
1812                 ret = -ENOENT;
1813                 goto out;
1814         }
1815         state->private = private;
1816 out:
1817         spin_unlock(&tree->lock);
1818         return ret;
1819 }
1820
1821 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1822 {
1823         struct rb_node *node;
1824         struct extent_state *state;
1825         int ret = 0;
1826
1827         spin_lock(&tree->lock);
1828         /*
1829          * this search will find all the extents that end after
1830          * our range starts.
1831          */
1832         node = tree_search(tree, start);
1833         if (!node) {
1834                 ret = -ENOENT;
1835                 goto out;
1836         }
1837         state = rb_entry(node, struct extent_state, rb_node);
1838         if (state->start != start) {
1839                 ret = -ENOENT;
1840                 goto out;
1841         }
1842         *private = state->private;
1843 out:
1844         spin_unlock(&tree->lock);
1845         return ret;
1846 }
1847
1848 /*
1849  * searches a range in the state tree for a given mask.
1850  * If 'filled' == 1, this returns 1 only if every extent in the tree
1851  * has the bits set.  Otherwise, 1 is returned if any bit in the
1852  * range is found set.
1853  */
1854 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1855                    unsigned long bits, int filled, struct extent_state *cached)
1856 {
1857         struct extent_state *state = NULL;
1858         struct rb_node *node;
1859         int bitset = 0;
1860
1861         spin_lock(&tree->lock);
1862         if (cached && cached->tree && cached->start <= start &&
1863             cached->end > start)
1864                 node = &cached->rb_node;
1865         else
1866                 node = tree_search(tree, start);
1867         while (node && start <= end) {
1868                 state = rb_entry(node, struct extent_state, rb_node);
1869
1870                 if (filled && state->start > start) {
1871                         bitset = 0;
1872                         break;
1873                 }
1874
1875                 if (state->start > end)
1876                         break;
1877
1878                 if (state->state & bits) {
1879                         bitset = 1;
1880                         if (!filled)
1881                                 break;
1882                 } else if (filled) {
1883                         bitset = 0;
1884                         break;
1885                 }
1886
1887                 if (state->end == (u64)-1)
1888                         break;
1889
1890                 start = state->end + 1;
1891                 if (start > end)
1892                         break;
1893                 node = rb_next(node);
1894                 if (!node) {
1895                         if (filled)
1896                                 bitset = 0;
1897                         break;
1898                 }
1899         }
1900         spin_unlock(&tree->lock);
1901         return bitset;
1902 }
1903
1904 /*
1905  * helper function to set a given page up to date if all the
1906  * extents in the tree for that page are up to date
1907  */
1908 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1909 {
1910         u64 start = page_offset(page);
1911         u64 end = start + PAGE_CACHE_SIZE - 1;
1912         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1913                 SetPageUptodate(page);
1914 }
1915
1916 /*
1917  * When IO fails, either with EIO or csum verification fails, we
1918  * try other mirrors that might have a good copy of the data.  This
1919  * io_failure_record is used to record state as we go through all the
1920  * mirrors.  If another mirror has good data, the page is set up to date
1921  * and things continue.  If a good mirror can't be found, the original
1922  * bio end_io callback is called to indicate things have failed.
1923  */
1924 struct io_failure_record {
1925         struct page *page;
1926         u64 start;
1927         u64 len;
1928         u64 logical;
1929         unsigned long bio_flags;
1930         int this_mirror;
1931         int failed_mirror;
1932         int in_validation;
1933 };
1934
1935 static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1936                                 int did_repair)
1937 {
1938         int ret;
1939         int err = 0;
1940         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1941
1942         set_state_private(failure_tree, rec->start, 0);
1943         ret = clear_extent_bits(failure_tree, rec->start,
1944                                 rec->start + rec->len - 1,
1945                                 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1946         if (ret)
1947                 err = ret;
1948
1949         ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1950                                 rec->start + rec->len - 1,
1951                                 EXTENT_DAMAGED, GFP_NOFS);
1952         if (ret && !err)
1953                 err = ret;
1954
1955         kfree(rec);
1956         return err;
1957 }
1958
1959 static void repair_io_failure_callback(struct bio *bio, int err)
1960 {
1961         complete(bio->bi_private);
1962 }
1963
1964 /*
1965  * this bypasses the standard btrfs submit functions deliberately, as
1966  * the standard behavior is to write all copies in a raid setup. here we only
1967  * want to write the one bad copy. so we do the mapping for ourselves and issue
1968  * submit_bio directly.
1969  * to avoid any synchronization issues, wait for the data after writing, which
1970  * actually prevents the read that triggered the error from finishing.
1971  * currently, there can be no more than two copies of every data bit. thus,
1972  * exactly one rewrite is required.
1973  */
1974 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
1975                         u64 length, u64 logical, struct page *page,
1976                         int mirror_num)
1977 {
1978         struct bio *bio;
1979         struct btrfs_device *dev;
1980         DECLARE_COMPLETION_ONSTACK(compl);
1981         u64 map_length = 0;
1982         u64 sector;
1983         struct btrfs_bio *bbio = NULL;
1984         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
1985         int ret;
1986
1987         BUG_ON(!mirror_num);
1988
1989         /* we can't repair anything in raid56 yet */
1990         if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
1991                 return 0;
1992
1993         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1994         if (!bio)
1995                 return -EIO;
1996         bio->bi_private = &compl;
1997         bio->bi_end_io = repair_io_failure_callback;
1998         bio->bi_size = 0;
1999         map_length = length;
2000
2001         ret = btrfs_map_block(fs_info, WRITE, logical,
2002                               &map_length, &bbio, mirror_num);
2003         if (ret) {
2004                 bio_put(bio);
2005                 return -EIO;
2006         }
2007         BUG_ON(mirror_num != bbio->mirror_num);
2008         sector = bbio->stripes[mirror_num-1].physical >> 9;
2009         bio->bi_sector = sector;
2010         dev = bbio->stripes[mirror_num-1].dev;
2011         kfree(bbio);
2012         if (!dev || !dev->bdev || !dev->writeable) {
2013                 bio_put(bio);
2014                 return -EIO;
2015         }
2016         bio->bi_bdev = dev->bdev;
2017         bio_add_page(bio, page, length, start - page_offset(page));
2018         btrfsic_submit_bio(WRITE_SYNC, bio);
2019         wait_for_completion(&compl);
2020
2021         if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2022                 /* try to remap that extent elsewhere? */
2023                 bio_put(bio);
2024                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2025                 return -EIO;
2026         }
2027
2028         printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
2029                       "(dev %s sector %llu)\n", page->mapping->host->i_ino,
2030                       start, rcu_str_deref(dev->name), sector);
2031
2032         bio_put(bio);
2033         return 0;
2034 }
2035
2036 int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2037                          int mirror_num)
2038 {
2039         u64 start = eb->start;
2040         unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
2041         int ret = 0;
2042
2043         for (i = 0; i < num_pages; i++) {
2044                 struct page *p = extent_buffer_page(eb, i);
2045                 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
2046                                         start, p, mirror_num);
2047                 if (ret)
2048                         break;
2049                 start += PAGE_CACHE_SIZE;
2050         }
2051
2052         return ret;
2053 }
2054
2055 /*
2056  * each time an IO finishes, we do a fast check in the IO failure tree
2057  * to see if we need to process or clean up an io_failure_record
2058  */
2059 static int clean_io_failure(u64 start, struct page *page)
2060 {
2061         u64 private;
2062         u64 private_failure;
2063         struct io_failure_record *failrec;
2064         struct btrfs_fs_info *fs_info;
2065         struct extent_state *state;
2066         int num_copies;
2067         int did_repair = 0;
2068         int ret;
2069         struct inode *inode = page->mapping->host;
2070
2071         private = 0;
2072         ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2073                                 (u64)-1, 1, EXTENT_DIRTY, 0);
2074         if (!ret)
2075                 return 0;
2076
2077         ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2078                                 &private_failure);
2079         if (ret)
2080                 return 0;
2081
2082         failrec = (struct io_failure_record *)(unsigned long) private_failure;
2083         BUG_ON(!failrec->this_mirror);
2084
2085         if (failrec->in_validation) {
2086                 /* there was no real error, just free the record */
2087                 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2088                          failrec->start);
2089                 did_repair = 1;
2090                 goto out;
2091         }
2092
2093         spin_lock(&BTRFS_I(inode)->io_tree.lock);
2094         state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2095                                             failrec->start,
2096                                             EXTENT_LOCKED);
2097         spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2098
2099         if (state && state->start <= failrec->start &&
2100             state->end >= failrec->start + failrec->len - 1) {
2101                 fs_info = BTRFS_I(inode)->root->fs_info;
2102                 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2103                                               failrec->len);
2104                 if (num_copies > 1)  {
2105                         ret = repair_io_failure(fs_info, start, failrec->len,
2106                                                 failrec->logical, page,
2107                                                 failrec->failed_mirror);
2108                         did_repair = !ret;
2109                 }
2110                 ret = 0;
2111         }
2112
2113 out:
2114         if (!ret)
2115                 ret = free_io_failure(inode, failrec, did_repair);
2116
2117         return ret;
2118 }
2119
2120 /*
2121  * this is a generic handler for readpage errors (default
2122  * readpage_io_failed_hook). if other copies exist, read those and write back
2123  * good data to the failed position. does not investigate in remapping the
2124  * failed extent elsewhere, hoping the device will be smart enough to do this as
2125  * needed
2126  */
2127
2128 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2129                               struct page *page, u64 start, u64 end,
2130                               int failed_mirror)
2131 {
2132         struct io_failure_record *failrec = NULL;
2133         u64 private;
2134         struct extent_map *em;
2135         struct inode *inode = page->mapping->host;
2136         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2137         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2138         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2139         struct bio *bio;
2140         struct btrfs_io_bio *btrfs_failed_bio;
2141         struct btrfs_io_bio *btrfs_bio;
2142         int num_copies;
2143         int ret;
2144         int read_mode;
2145         u64 logical;
2146
2147         BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2148
2149         ret = get_state_private(failure_tree, start, &private);
2150         if (ret) {
2151                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2152                 if (!failrec)
2153                         return -ENOMEM;
2154                 failrec->start = start;
2155                 failrec->len = end - start + 1;
2156                 failrec->this_mirror = 0;
2157                 failrec->bio_flags = 0;
2158                 failrec->in_validation = 0;
2159
2160                 read_lock(&em_tree->lock);
2161                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2162                 if (!em) {
2163                         read_unlock(&em_tree->lock);
2164                         kfree(failrec);
2165                         return -EIO;
2166                 }
2167
2168                 if (em->start > start || em->start + em->len < start) {
2169                         free_extent_map(em);
2170                         em = NULL;
2171                 }
2172                 read_unlock(&em_tree->lock);
2173
2174                 if (!em) {
2175                         kfree(failrec);
2176                         return -EIO;
2177                 }
2178                 logical = start - em->start;
2179                 logical = em->block_start + logical;
2180                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2181                         logical = em->block_start;
2182                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2183                         extent_set_compress_type(&failrec->bio_flags,
2184                                                  em->compress_type);
2185                 }
2186                 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2187                          "len=%llu\n", logical, start, failrec->len);
2188                 failrec->logical = logical;
2189                 free_extent_map(em);
2190
2191                 /* set the bits in the private failure tree */
2192                 ret = set_extent_bits(failure_tree, start, end,
2193                                         EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2194                 if (ret >= 0)
2195                         ret = set_state_private(failure_tree, start,
2196                                                 (u64)(unsigned long)failrec);
2197                 /* set the bits in the inode's tree */
2198                 if (ret >= 0)
2199                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2200                                                 GFP_NOFS);
2201                 if (ret < 0) {
2202                         kfree(failrec);
2203                         return ret;
2204                 }
2205         } else {
2206                 failrec = (struct io_failure_record *)(unsigned long)private;
2207                 pr_debug("bio_readpage_error: (found) logical=%llu, "
2208                          "start=%llu, len=%llu, validation=%d\n",
2209                          failrec->logical, failrec->start, failrec->len,
2210                          failrec->in_validation);
2211                 /*
2212                  * when data can be on disk more than twice, add to failrec here
2213                  * (e.g. with a list for failed_mirror) to make
2214                  * clean_io_failure() clean all those errors at once.
2215                  */
2216         }
2217         num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2218                                       failrec->logical, failrec->len);
2219         if (num_copies == 1) {
2220                 /*
2221                  * we only have a single copy of the data, so don't bother with
2222                  * all the retry and error correction code that follows. no
2223                  * matter what the error is, it is very likely to persist.
2224                  */
2225                 pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
2226                          num_copies, failrec->this_mirror, failed_mirror);
2227                 free_io_failure(inode, failrec, 0);
2228                 return -EIO;
2229         }
2230
2231         /*
2232          * there are two premises:
2233          *      a) deliver good data to the caller
2234          *      b) correct the bad sectors on disk
2235          */
2236         if (failed_bio->bi_vcnt > 1) {
2237                 /*
2238                  * to fulfill b), we need to know the exact failing sectors, as
2239                  * we don't want to rewrite any more than the failed ones. thus,
2240                  * we need separate read requests for the failed bio
2241                  *
2242                  * if the following BUG_ON triggers, our validation request got
2243                  * merged. we need separate requests for our algorithm to work.
2244                  */
2245                 BUG_ON(failrec->in_validation);
2246                 failrec->in_validation = 1;
2247                 failrec->this_mirror = failed_mirror;
2248                 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2249         } else {
2250                 /*
2251                  * we're ready to fulfill a) and b) alongside. get a good copy
2252                  * of the failed sector and if we succeed, we have setup
2253                  * everything for repair_io_failure to do the rest for us.
2254                  */
2255                 if (failrec->in_validation) {
2256                         BUG_ON(failrec->this_mirror != failed_mirror);
2257                         failrec->in_validation = 0;
2258                         failrec->this_mirror = 0;
2259                 }
2260                 failrec->failed_mirror = failed_mirror;
2261                 failrec->this_mirror++;
2262                 if (failrec->this_mirror == failed_mirror)
2263                         failrec->this_mirror++;
2264                 read_mode = READ_SYNC;
2265         }
2266
2267         if (failrec->this_mirror > num_copies) {
2268                 pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
2269                          num_copies, failrec->this_mirror, failed_mirror);
2270                 free_io_failure(inode, failrec, 0);
2271                 return -EIO;
2272         }
2273
2274         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2275         if (!bio) {
2276                 free_io_failure(inode, failrec, 0);
2277                 return -EIO;
2278         }
2279         bio->bi_end_io = failed_bio->bi_end_io;
2280         bio->bi_sector = failrec->logical >> 9;
2281         bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2282         bio->bi_size = 0;
2283
2284         btrfs_failed_bio = btrfs_io_bio(failed_bio);
2285         if (btrfs_failed_bio->csum) {
2286                 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2287                 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2288
2289                 btrfs_bio = btrfs_io_bio(bio);
2290                 btrfs_bio->csum = btrfs_bio->csum_inline;
2291                 phy_offset >>= inode->i_sb->s_blocksize_bits;
2292                 phy_offset *= csum_size;
2293                 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset,
2294                        csum_size);
2295         }
2296
2297         bio_add_page(bio, page, failrec->len, start - page_offset(page));
2298
2299         pr_debug("bio_readpage_error: submitting new read[%#x] to "
2300                  "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2301                  failrec->this_mirror, num_copies, failrec->in_validation);
2302
2303         ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2304                                          failrec->this_mirror,
2305                                          failrec->bio_flags, 0);
2306         return ret;
2307 }
2308
2309 /* lots and lots of room for performance fixes in the end_bio funcs */
2310
2311 int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2312 {
2313         int uptodate = (err == 0);
2314         struct extent_io_tree *tree;
2315         int ret;
2316
2317         tree = &BTRFS_I(page->mapping->host)->io_tree;
2318
2319         if (tree->ops && tree->ops->writepage_end_io_hook) {
2320                 ret = tree->ops->writepage_end_io_hook(page, start,
2321                                                end, NULL, uptodate);
2322                 if (ret)
2323                         uptodate = 0;
2324         }
2325
2326         if (!uptodate) {
2327                 ClearPageUptodate(page);
2328                 SetPageError(page);
2329         }
2330         return 0;
2331 }
2332
2333 /*
2334  * after a writepage IO is done, we need to:
2335  * clear the uptodate bits on error
2336  * clear the writeback bits in the extent tree for this IO
2337  * end_page_writeback if the page has no more pending IO
2338  *
2339  * Scheduling is not allowed, so the extent state tree is expected
2340  * to have one and only one object corresponding to this IO.
2341  */
2342 static void end_bio_extent_writepage(struct bio *bio, int err)
2343 {
2344         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2345         struct extent_io_tree *tree;
2346         u64 start;
2347         u64 end;
2348
2349         do {
2350                 struct page *page = bvec->bv_page;
2351                 tree = &BTRFS_I(page->mapping->host)->io_tree;
2352
2353                 /* We always issue full-page reads, but if some block
2354                  * in a page fails to read, blk_update_request() will
2355                  * advance bv_offset and adjust bv_len to compensate.
2356                  * Print a warning for nonzero offsets, and an error
2357                  * if they don't add up to a full page.  */
2358                 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2359                         printk("%s page write in btrfs with offset %u and length %u\n",
2360                                bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2361                                ? KERN_ERR "partial" : KERN_INFO "incomplete",
2362                                bvec->bv_offset, bvec->bv_len);
2363
2364                 start = page_offset(page);
2365                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2366
2367                 if (--bvec >= bio->bi_io_vec)
2368                         prefetchw(&bvec->bv_page->flags);
2369
2370                 if (end_extent_writepage(page, err, start, end))
2371                         continue;
2372
2373                 end_page_writeback(page);
2374         } while (bvec >= bio->bi_io_vec);
2375
2376         bio_put(bio);
2377 }
2378
2379 static void
2380 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2381                               int uptodate)
2382 {
2383         struct extent_state *cached = NULL;
2384         u64 end = start + len - 1;
2385
2386         if (uptodate && tree->track_uptodate)
2387                 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2388         unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2389 }
2390
2391 /*
2392  * after a readpage IO is done, we need to:
2393  * clear the uptodate bits on error
2394  * set the uptodate bits if things worked
2395  * set the page up to date if all extents in the tree are uptodate
2396  * clear the lock bit in the extent tree
2397  * unlock the page if there are no other extents locked for it
2398  *
2399  * Scheduling is not allowed, so the extent state tree is expected
2400  * to have one and only one object corresponding to this IO.
2401  */
2402 static void end_bio_extent_readpage(struct bio *bio, int err)
2403 {
2404         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
2405         struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2406         struct bio_vec *bvec = bio->bi_io_vec;
2407         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2408         struct extent_io_tree *tree;
2409         u64 offset = 0;
2410         u64 start;
2411         u64 end;
2412         u64 len;
2413         u64 extent_start = 0;
2414         u64 extent_len = 0;
2415         int mirror;
2416         int ret;
2417
2418         if (err)
2419                 uptodate = 0;
2420
2421         do {
2422                 struct page *page = bvec->bv_page;
2423                 struct inode *inode = page->mapping->host;
2424
2425                 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2426                          "mirror=%lu\n", (u64)bio->bi_sector, err,
2427                          io_bio->mirror_num);
2428                 tree = &BTRFS_I(inode)->io_tree;
2429
2430                 /* We always issue full-page reads, but if some block
2431                  * in a page fails to read, blk_update_request() will
2432                  * advance bv_offset and adjust bv_len to compensate.
2433                  * Print a warning for nonzero offsets, and an error
2434                  * if they don't add up to a full page.  */
2435                 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2436                         printk("%s page read in btrfs with offset %u and length %u\n",
2437                                bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2438                                ? KERN_ERR "partial" : KERN_INFO "incomplete",
2439                                bvec->bv_offset, bvec->bv_len);
2440
2441                 start = page_offset(page);
2442                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2443                 len = bvec->bv_len;
2444
2445                 if (++bvec <= bvec_end)
2446                         prefetchw(&bvec->bv_page->flags);
2447
2448                 mirror = io_bio->mirror_num;
2449                 if (likely(uptodate && tree->ops &&
2450                            tree->ops->readpage_end_io_hook)) {
2451                         ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2452                                                               page, start, end,
2453                                                               mirror);
2454                         if (ret)
2455                                 uptodate = 0;
2456                         else
2457                                 clean_io_failure(start, page);
2458                 }
2459
2460                 if (likely(uptodate))
2461                         goto readpage_ok;
2462
2463                 if (tree->ops && tree->ops->readpage_io_failed_hook) {
2464                         ret = tree->ops->readpage_io_failed_hook(page, mirror);
2465                         if (!ret && !err &&
2466                             test_bit(BIO_UPTODATE, &bio->bi_flags))
2467                                 uptodate = 1;
2468                 } else {
2469                         /*
2470                          * The generic bio_readpage_error handles errors the
2471                          * following way: If possible, new read requests are
2472                          * created and submitted and will end up in
2473                          * end_bio_extent_readpage as well (if we're lucky, not
2474                          * in the !uptodate case). In that case it returns 0 and
2475                          * we just go on with the next page in our bio. If it
2476                          * can't handle the error it will return -EIO and we
2477                          * remain responsible for that page.
2478                          */
2479                         ret = bio_readpage_error(bio, offset, page, start, end,
2480                                                  mirror);
2481                         if (ret == 0) {
2482                                 uptodate =
2483                                         test_bit(BIO_UPTODATE, &bio->bi_flags);
2484                                 if (err)
2485                                         uptodate = 0;
2486                                 continue;
2487                         }
2488                 }
2489 readpage_ok:
2490                 if (likely(uptodate)) {
2491                         loff_t i_size = i_size_read(inode);
2492                         pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2493                         unsigned offset;
2494
2495                         /* Zero out the end if this page straddles i_size */
2496                         offset = i_size & (PAGE_CACHE_SIZE-1);
2497                         if (page->index == end_index && offset)
2498                                 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2499                         SetPageUptodate(page);
2500                 } else {
2501                         ClearPageUptodate(page);
2502                         SetPageError(page);
2503                 }
2504                 unlock_page(page);
2505                 offset += len;
2506
2507                 if (unlikely(!uptodate)) {
2508                         if (extent_len) {
2509                                 endio_readpage_release_extent(tree,
2510                                                               extent_start,
2511                                                               extent_len, 1);
2512                                 extent_start = 0;
2513                                 extent_len = 0;
2514                         }
2515                         endio_readpage_release_extent(tree, start,
2516                                                       end - start + 1, 0);
2517                 } else if (!extent_len) {
2518                         extent_start = start;
2519                         extent_len = end + 1 - start;
2520                 } else if (extent_start + extent_len == start) {
2521                         extent_len += end + 1 - start;
2522                 } else {
2523                         endio_readpage_release_extent(tree, extent_start,
2524                                                       extent_len, uptodate);
2525                         extent_start = start;
2526                         extent_len = end + 1 - start;
2527                 }
2528         } while (bvec <= bvec_end);
2529
2530         if (extent_len)
2531                 endio_readpage_release_extent(tree, extent_start, extent_len,
2532                                               uptodate);
2533         if (io_bio->end_io)
2534                 io_bio->end_io(io_bio, err);
2535         bio_put(bio);
2536 }
2537
2538 /*
2539  * this allocates from the btrfs_bioset.  We're returning a bio right now
2540  * but you can call btrfs_io_bio for the appropriate container_of magic
2541  */
2542 struct bio *
2543 btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2544                 gfp_t gfp_flags)
2545 {
2546         struct btrfs_io_bio *btrfs_bio;
2547         struct bio *bio;
2548
2549         bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
2550
2551         if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2552                 while (!bio && (nr_vecs /= 2)) {
2553                         bio = bio_alloc_bioset(gfp_flags,
2554                                                nr_vecs, btrfs_bioset);
2555                 }
2556         }
2557
2558         if (bio) {
2559                 bio->bi_size = 0;
2560                 bio->bi_bdev = bdev;
2561                 bio->bi_sector = first_sector;
2562                 btrfs_bio = btrfs_io_bio(bio);
2563                 btrfs_bio->csum = NULL;
2564                 btrfs_bio->csum_allocated = NULL;
2565                 btrfs_bio->end_io = NULL;
2566         }
2567         return bio;
2568 }
2569
2570 struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2571 {
2572         return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2573 }
2574
2575
2576 /* this also allocates from the btrfs_bioset */
2577 struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2578 {
2579         struct btrfs_io_bio *btrfs_bio;
2580         struct bio *bio;
2581
2582         bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2583         if (bio) {
2584                 btrfs_bio = btrfs_io_bio(bio);
2585                 btrfs_bio->csum = NULL;
2586                 btrfs_bio->csum_allocated = NULL;
2587                 btrfs_bio->end_io = NULL;
2588         }
2589         return bio;
2590 }
2591
2592
2593 static int __must_check submit_one_bio(int rw, struct bio *bio,
2594                                        int mirror_num, unsigned long bio_flags)
2595 {
2596         int ret = 0;
2597         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2598         struct page *page = bvec->bv_page;
2599         struct extent_io_tree *tree = bio->bi_private;
2600         u64 start;
2601
2602         start = page_offset(page) + bvec->bv_offset;
2603
2604         bio->bi_private = NULL;
2605
2606         bio_get(bio);
2607
2608         if (tree->ops && tree->ops->submit_bio_hook)
2609                 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
2610                                            mirror_num, bio_flags, start);
2611         else
2612                 btrfsic_submit_bio(rw, bio);
2613
2614         if (bio_flagged(bio, BIO_EOPNOTSUPP))
2615                 ret = -EOPNOTSUPP;
2616         bio_put(bio);
2617         return ret;
2618 }
2619
2620 static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
2621                      unsigned long offset, size_t size, struct bio *bio,
2622                      unsigned long bio_flags)
2623 {
2624         int ret = 0;
2625         if (tree->ops && tree->ops->merge_bio_hook)
2626                 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
2627                                                 bio_flags);
2628         BUG_ON(ret < 0);
2629         return ret;
2630
2631 }
2632
2633 static int submit_extent_page(int rw, struct extent_io_tree *tree,
2634                               struct page *page, sector_t sector,
2635                               size_t size, unsigned long offset,
2636                               struct block_device *bdev,
2637                               struct bio **bio_ret,
2638                               unsigned long max_pages,
2639                               bio_end_io_t end_io_func,
2640                               int mirror_num,
2641                               unsigned long prev_bio_flags,
2642                               unsigned long bio_flags)
2643 {
2644         int ret = 0;
2645         struct bio *bio;
2646         int nr;
2647         int contig = 0;
2648         int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2649         int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2650         size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
2651
2652         if (bio_ret && *bio_ret) {
2653                 bio = *bio_ret;
2654                 if (old_compressed)
2655                         contig = bio->bi_sector == sector;
2656                 else
2657                         contig = bio_end_sector(bio) == sector;
2658
2659                 if (prev_bio_flags != bio_flags || !contig ||
2660                     merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
2661                     bio_add_page(bio, page, page_size, offset) < page_size) {
2662                         ret = submit_one_bio(rw, bio, mirror_num,
2663                                              prev_bio_flags);
2664                         if (ret < 0)
2665                                 return ret;
2666                         bio = NULL;
2667                 } else {
2668                         return 0;
2669                 }
2670         }
2671         if (this_compressed)
2672                 nr = BIO_MAX_PAGES;
2673         else
2674                 nr = bio_get_nr_vecs(bdev);
2675
2676         bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
2677         if (!bio)
2678                 return -ENOMEM;
2679
2680         bio_add_page(bio, page, page_size, offset);
2681         bio->bi_end_io = end_io_func;
2682         bio->bi_private = tree;
2683
2684         if (bio_ret)
2685                 *bio_ret = bio;
2686         else
2687                 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
2688
2689         return ret;
2690 }
2691
2692 static void attach_extent_buffer_page(struct extent_buffer *eb,
2693                                       struct page *page)
2694 {
2695         if (!PagePrivate(page)) {
2696                 SetPagePrivate(page);
2697                 page_cache_get(page);
2698                 set_page_private(page, (unsigned long)eb);
2699         } else {
2700                 WARN_ON(page->private != (unsigned long)eb);
2701         }
2702 }
2703
2704 void set_page_extent_mapped(struct page *page)
2705 {
2706         if (!PagePrivate(page)) {
2707                 SetPagePrivate(page);
2708                 page_cache_get(page);
2709                 set_page_private(page, EXTENT_PAGE_PRIVATE);
2710         }
2711 }
2712
2713 static struct extent_map *
2714 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2715                  u64 start, u64 len, get_extent_t *get_extent,
2716                  struct extent_map **em_cached)
2717 {
2718         struct extent_map *em;
2719
2720         if (em_cached && *em_cached) {
2721                 em = *em_cached;
2722                 if (em->in_tree && start >= em->start &&
2723                     start < extent_map_end(em)) {
2724                         atomic_inc(&em->refs);
2725                         return em;
2726                 }
2727
2728                 free_extent_map(em);
2729                 *em_cached = NULL;
2730         }
2731
2732         em = get_extent(inode, page, pg_offset, start, len, 0);
2733         if (em_cached && !IS_ERR_OR_NULL(em)) {
2734                 BUG_ON(*em_cached);
2735                 atomic_inc(&em->refs);
2736                 *em_cached = em;
2737         }
2738         return em;
2739 }
2740 /*
2741  * basic readpage implementation.  Locked extent state structs are inserted
2742  * into the tree that are removed when the IO is done (by the end_io
2743  * handlers)
2744  * XXX JDM: This needs looking at to ensure proper page locking
2745  */
2746 static int __do_readpage(struct extent_io_tree *tree,
2747                          struct page *page,
2748                          get_extent_t *get_extent,
2749                          struct extent_map **em_cached,
2750                          struct bio **bio, int mirror_num,
2751                          unsigned long *bio_flags, int rw)
2752 {
2753         struct inode *inode = page->mapping->host;
2754         u64 start = page_offset(page);
2755         u64 page_end = start + PAGE_CACHE_SIZE - 1;
2756         u64 end;
2757         u64 cur = start;
2758         u64 extent_offset;
2759         u64 last_byte = i_size_read(inode);
2760         u64 block_start;
2761         u64 cur_end;
2762         sector_t sector;
2763         struct extent_map *em;
2764         struct block_device *bdev;
2765         int ret;
2766         int nr = 0;
2767         int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2768         size_t pg_offset = 0;
2769         size_t iosize;
2770         size_t disk_io_size;
2771         size_t blocksize = inode->i_sb->s_blocksize;
2772         unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2773
2774         set_page_extent_mapped(page);
2775
2776         end = page_end;
2777         if (!PageUptodate(page)) {
2778                 if (cleancache_get_page(page) == 0) {
2779                         BUG_ON(blocksize != PAGE_SIZE);
2780                         unlock_extent(tree, start, end);
2781                         goto out;
2782                 }
2783         }
2784
2785         if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2786                 char *userpage;
2787                 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2788
2789                 if (zero_offset) {
2790                         iosize = PAGE_CACHE_SIZE - zero_offset;
2791                         userpage = kmap_atomic(page);
2792                         memset(userpage + zero_offset, 0, iosize);
2793                         flush_dcache_page(page);
2794                         kunmap_atomic(userpage);
2795                 }
2796         }
2797         while (cur <= end) {
2798                 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2799
2800                 if (cur >= last_byte) {
2801                         char *userpage;
2802                         struct extent_state *cached = NULL;
2803
2804                         iosize = PAGE_CACHE_SIZE - pg_offset;
2805                         userpage = kmap_atomic(page);
2806                         memset(userpage + pg_offset, 0, iosize);
2807                         flush_dcache_page(page);
2808                         kunmap_atomic(userpage);
2809                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2810                                             &cached, GFP_NOFS);
2811                         if (!parent_locked)
2812                                 unlock_extent_cached(tree, cur,
2813                                                      cur + iosize - 1,
2814                                                      &cached, GFP_NOFS);
2815                         break;
2816                 }
2817                 em = __get_extent_map(inode, page, pg_offset, cur,
2818                                       end - cur + 1, get_extent, em_cached);
2819                 if (IS_ERR_OR_NULL(em)) {
2820                         SetPageError(page);
2821                         if (!parent_locked)
2822                                 unlock_extent(tree, cur, end);
2823                         break;
2824                 }
2825                 extent_offset = cur - em->start;
2826                 BUG_ON(extent_map_end(em) <= cur);
2827                 BUG_ON(end < cur);
2828
2829                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2830                         this_bio_flag |= EXTENT_BIO_COMPRESSED;
2831                         extent_set_compress_type(&this_bio_flag,
2832                                                  em->compress_type);
2833                 }
2834
2835                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2836                 cur_end = min(extent_map_end(em) - 1, end);
2837                 iosize = ALIGN(iosize, blocksize);
2838                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2839                         disk_io_size = em->block_len;
2840                         sector = em->block_start >> 9;
2841                 } else {
2842                         sector = (em->block_start + extent_offset) >> 9;
2843                         disk_io_size = iosize;
2844                 }
2845                 bdev = em->bdev;
2846                 block_start = em->block_start;
2847                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2848                         block_start = EXTENT_MAP_HOLE;
2849                 free_extent_map(em);
2850                 em = NULL;
2851
2852                 /* we've found a hole, just zero and go on */
2853                 if (block_start == EXTENT_MAP_HOLE) {
2854                         char *userpage;
2855                         struct extent_state *cached = NULL;
2856
2857                         userpage = kmap_atomic(page);
2858                         memset(userpage + pg_offset, 0, iosize);
2859                         flush_dcache_page(page);
2860                         kunmap_atomic(userpage);
2861
2862                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2863                                             &cached, GFP_NOFS);
2864                         unlock_extent_cached(tree, cur, cur + iosize - 1,
2865                                              &cached, GFP_NOFS);
2866                         cur = cur + iosize;
2867                         pg_offset += iosize;
2868                         continue;
2869                 }
2870                 /* the get_extent function already copied into the page */
2871                 if (test_range_bit(tree, cur, cur_end,
2872                                    EXTENT_UPTODATE, 1, NULL)) {
2873                         check_page_uptodate(tree, page);
2874                         if (!parent_locked)
2875                                 unlock_extent(tree, cur, cur + iosize - 1);
2876                         cur = cur + iosize;
2877                         pg_offset += iosize;
2878                         continue;
2879                 }
2880                 /* we have an inline extent but it didn't get marked up
2881                  * to date.  Error out
2882                  */
2883                 if (block_start == EXTENT_MAP_INLINE) {
2884                         SetPageError(page);
2885                         if (!parent_locked)
2886                                 unlock_extent(tree, cur, cur + iosize - 1);
2887                         cur = cur + iosize;
2888                         pg_offset += iosize;
2889                         continue;
2890                 }
2891
2892                 pnr -= page->index;
2893                 ret = submit_extent_page(rw, tree, page,
2894                                          sector, disk_io_size, pg_offset,
2895                                          bdev, bio, pnr,
2896                                          end_bio_extent_readpage, mirror_num,
2897                                          *bio_flags,
2898                                          this_bio_flag);
2899                 if (!ret) {
2900                         nr++;
2901                         *bio_flags = this_bio_flag;
2902                 } else {
2903                         SetPageError(page);
2904                         if (!parent_locked)
2905                                 unlock_extent(tree, cur, cur + iosize - 1);
2906                 }
2907                 cur = cur + iosize;
2908                 pg_offset += iosize;
2909         }
2910 out:
2911         if (!nr) {
2912                 if (!PageError(page))
2913                         SetPageUptodate(page);
2914                 unlock_page(page);
2915         }
2916         return 0;
2917 }
2918
2919 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
2920                                              struct page *pages[], int nr_pages,
2921                                              u64 start, u64 end,
2922                                              get_extent_t *get_extent,
2923                                              struct extent_map **em_cached,
2924                                              struct bio **bio, int mirror_num,
2925                                              unsigned long *bio_flags, int rw)
2926 {
2927         struct inode *inode;
2928         struct btrfs_ordered_extent *ordered;
2929         int index;
2930
2931         inode = pages[0]->mapping->host;
2932         while (1) {
2933                 lock_extent(tree, start, end);
2934                 ordered = btrfs_lookup_ordered_range(inode, start,
2935                                                      end - start + 1);
2936                 if (!ordered)
2937                         break;
2938                 unlock_extent(tree, start, end);
2939                 btrfs_start_ordered_extent(inode, ordered, 1);
2940                 btrfs_put_ordered_extent(ordered);
2941         }
2942
2943         for (index = 0; index < nr_pages; index++) {
2944                 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
2945                               mirror_num, bio_flags, rw);
2946                 page_cache_release(pages[index]);
2947         }
2948 }
2949
2950 static void __extent_readpages(struct extent_io_tree *tree,
2951                                struct page *pages[],
2952                                int nr_pages, get_extent_t *get_extent,
2953                                struct extent_map **em_cached,
2954                                struct bio **bio, int mirror_num,
2955                                unsigned long *bio_flags, int rw)
2956 {
2957         u64 start = 0;
2958         u64 end = 0;
2959         u64 page_start;
2960         int index;
2961         int first_index = 0;
2962
2963         for (index = 0; index < nr_pages; index++) {
2964                 page_start = page_offset(pages[index]);
2965                 if (!end) {
2966                         start = page_start;
2967                         end = start + PAGE_CACHE_SIZE - 1;
2968                         first_index = index;
2969                 } else if (end + 1 == page_start) {
2970                         end += PAGE_CACHE_SIZE;
2971                 } else {
2972                         __do_contiguous_readpages(tree, &pages[first_index],
2973                                                   index - first_index, start,
2974                                                   end, get_extent, em_cached,
2975                                                   bio, mirror_num, bio_flags,
2976                                                   rw);
2977                         start = page_start;
2978                         end = start + PAGE_CACHE_SIZE - 1;
2979                         first_index = index;
2980                 }
2981         }
2982
2983         if (end)
2984                 __do_contiguous_readpages(tree, &pages[first_index],
2985                                           index - first_index, start,
2986                                           end, get_extent, em_cached, bio,
2987                                           mirror_num, bio_flags, rw);
2988 }
2989
2990 static int __extent_read_full_page(struct extent_io_tree *tree,
2991                                    struct page *page,
2992                                    get_extent_t *get_extent,
2993                                    struct bio **bio, int mirror_num,
2994                                    unsigned long *bio_flags, int rw)
2995 {
2996         struct inode *inode = page->mapping->host;
2997         struct btrfs_ordered_extent *ordered;
2998         u64 start = page_offset(page);
2999         u64 end = start + PAGE_CACHE_SIZE - 1;
3000         int ret;
3001
3002         while (1) {
3003                 lock_extent(tree, start, end);
3004                 ordered = btrfs_lookup_ordered_extent(inode, start);
3005                 if (!ordered)
3006                         break;
3007                 unlock_extent(tree, start, end);
3008                 btrfs_start_ordered_extent(inode, ordered, 1);
3009                 btrfs_put_ordered_extent(ordered);
3010         }
3011
3012         ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3013                             bio_flags, rw);
3014         return ret;
3015 }
3016
3017 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
3018                             get_extent_t *get_extent, int mirror_num)
3019 {
3020         struct bio *bio = NULL;
3021         unsigned long bio_flags = 0;
3022         int ret;
3023
3024         ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
3025                                       &bio_flags, READ);
3026         if (bio)
3027                 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3028         return ret;
3029 }
3030
3031 int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3032                                  get_extent_t *get_extent, int mirror_num)
3033 {
3034         struct bio *bio = NULL;
3035         unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3036         int ret;
3037
3038         ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
3039                                       &bio_flags, READ);
3040         if (bio)
3041                 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3042         return ret;
3043 }
3044
3045 static noinline void update_nr_written(struct page *page,
3046                                       struct writeback_control *wbc,
3047                                       unsigned long nr_written)
3048 {
3049         wbc->nr_to_write -= nr_written;
3050         if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3051             wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3052                 page->mapping->writeback_index = page->index + nr_written;
3053 }
3054
3055 /*
3056  * the writepage semantics are similar to regular writepage.  extent
3057  * records are inserted to lock ranges in the tree, and as dirty areas
3058  * are found, they are marked writeback.  Then the lock bits are removed
3059  * and the end_io handler clears the writeback ranges
3060  */
3061 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3062                               void *data)
3063 {
3064         struct inode *inode = page->mapping->host;
3065         struct extent_page_data *epd = data;
3066         struct extent_io_tree *tree = epd->tree;
3067         u64 start = page_offset(page);
3068         u64 delalloc_start;
3069         u64 page_end = start + PAGE_CACHE_SIZE - 1;
3070         u64 end;
3071         u64 cur = start;
3072         u64 extent_offset;
3073         u64 last_byte = i_size_read(inode);
3074         u64 block_start;
3075         u64 iosize;
3076         sector_t sector;
3077         struct extent_state *cached_state = NULL;
3078         struct extent_map *em;
3079         struct block_device *bdev;
3080         int ret;
3081         int nr = 0;
3082         size_t pg_offset = 0;
3083         size_t blocksize;
3084         loff_t i_size = i_size_read(inode);
3085         unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3086         u64 nr_delalloc;
3087         u64 delalloc_end;
3088         int page_started;
3089         int compressed;
3090         int write_flags;
3091         unsigned long nr_written = 0;
3092         bool fill_delalloc = true;
3093
3094         if (wbc->sync_mode == WB_SYNC_ALL)
3095                 write_flags = WRITE_SYNC;
3096         else
3097                 write_flags = WRITE;
3098
3099         trace___extent_writepage(page, inode, wbc);
3100
3101         WARN_ON(!PageLocked(page));
3102
3103         ClearPageError(page);
3104
3105         pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3106         if (page->index > end_index ||
3107            (page->index == end_index && !pg_offset)) {
3108                 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3109                 unlock_page(page);
3110                 return 0;
3111         }
3112
3113         if (page->index == end_index) {
3114                 char *userpage;
3115
3116                 userpage = kmap_atomic(page);
3117                 memset(userpage + pg_offset, 0,
3118                        PAGE_CACHE_SIZE - pg_offset);
3119                 kunmap_atomic(userpage);
3120                 flush_dcache_page(page);
3121         }
3122         pg_offset = 0;
3123
3124         set_page_extent_mapped(page);
3125
3126         if (!tree->ops || !tree->ops->fill_delalloc)
3127                 fill_delalloc = false;
3128
3129         delalloc_start = start;
3130         delalloc_end = 0;
3131         page_started = 0;
3132         if (!epd->extent_locked && fill_delalloc) {
3133                 u64 delalloc_to_write = 0;
3134                 /*
3135                  * make sure the wbc mapping index is at least updated
3136                  * to this page.
3137                  */
3138                 update_nr_written(page, wbc, 0);
3139
3140                 while (delalloc_end < page_end) {
3141                         nr_delalloc = find_lock_delalloc_range(inode, tree,
3142                                                        page,
3143                                                        &delalloc_start,
3144                                                        &delalloc_end,
3145                                                        128 * 1024 * 1024);
3146                         if (nr_delalloc == 0) {
3147                                 delalloc_start = delalloc_end + 1;
3148                                 continue;
3149                         }
3150                         ret = tree->ops->fill_delalloc(inode, page,
3151                                                        delalloc_start,
3152                                                        delalloc_end,
3153                                                        &page_started,
3154                                                        &nr_written);
3155                         /* File system has been set read-only */
3156                         if (ret) {
3157                                 SetPageError(page);
3158                                 goto done;
3159                         }
3160                         /*
3161                          * delalloc_end is already one less than the total
3162                          * length, so we don't subtract one from
3163                          * PAGE_CACHE_SIZE
3164                          */
3165                         delalloc_to_write += (delalloc_end - delalloc_start +
3166                                               PAGE_CACHE_SIZE) >>
3167                                               PAGE_CACHE_SHIFT;
3168                         delalloc_start = delalloc_end + 1;
3169                 }
3170                 if (wbc->nr_to_write < delalloc_to_write) {
3171                         int thresh = 8192;
3172
3173                         if (delalloc_to_write < thresh * 2)
3174                                 thresh = delalloc_to_write;
3175                         wbc->nr_to_write = min_t(u64, delalloc_to_write,
3176                                                  thresh);
3177                 }
3178
3179                 /* did the fill delalloc function already unlock and start
3180                  * the IO?
3181                  */
3182                 if (page_started) {
3183                         ret = 0;
3184                         /*
3185                          * we've unlocked the page, so we can't update
3186                          * the mapping's writeback index, just update
3187                          * nr_to_write.
3188                          */
3189                         wbc->nr_to_write -= nr_written;
3190                         goto done_unlocked;
3191                 }
3192         }
3193         if (tree->ops && tree->ops->writepage_start_hook) {
3194                 ret = tree->ops->writepage_start_hook(page, start,
3195                                                       page_end);
3196                 if (ret) {
3197                         /* Fixup worker will requeue */
3198                         if (ret == -EBUSY)
3199                                 wbc->pages_skipped++;
3200                         else
3201                                 redirty_page_for_writepage(wbc, page);
3202                         update_nr_written(page, wbc, nr_written);
3203                         unlock_page(page);
3204                         ret = 0;
3205                         goto done_unlocked;
3206                 }
3207         }
3208
3209         /*
3210          * we don't want to touch the inode after unlocking the page,
3211          * so we update the mapping writeback index now
3212          */
3213         update_nr_written(page, wbc, nr_written + 1);
3214
3215         end = page_end;
3216         if (last_byte <= start) {
3217                 if (tree->ops && tree->ops->writepage_end_io_hook)
3218                         tree->ops->writepage_end_io_hook(page, start,
3219                                                          page_end, NULL, 1);
3220                 goto done;
3221         }
3222
3223         blocksize = inode->i_sb->s_blocksize;
3224
3225         while (cur <= end) {
3226                 if (cur >= last_byte) {
3227                         if (tree->ops && tree->ops->writepage_end_io_hook)
3228                                 tree->ops->writepage_end_io_hook(page, cur,
3229                                                          page_end, NULL, 1);
3230                         break;
3231                 }
3232                 em = epd->get_extent(inode, page, pg_offset, cur,
3233                                      end - cur + 1, 1);
3234                 if (IS_ERR_OR_NULL(em)) {
3235                         SetPageError(page);
3236                         break;
3237                 }
3238
3239                 extent_offset = cur - em->start;
3240                 BUG_ON(extent_map_end(em) <= cur);
3241                 BUG_ON(end < cur);
3242                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3243                 iosize = ALIGN(iosize, blocksize);
3244                 sector = (em->block_start + extent_offset) >> 9;
3245                 bdev = em->bdev;
3246                 block_start = em->block_start;
3247                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
3248                 free_extent_map(em);
3249                 em = NULL;
3250
3251                 /*
3252                  * compressed and inline extents are written through other
3253                  * paths in the FS
3254                  */
3255                 if (compressed || block_start == EXTENT_MAP_HOLE ||
3256                     block_start == EXTENT_MAP_INLINE) {
3257                         /*
3258                          * end_io notification does not happen here for
3259                          * compressed extents
3260                          */
3261                         if (!compressed && tree->ops &&
3262                             tree->ops->writepage_end_io_hook)
3263                                 tree->ops->writepage_end_io_hook(page, cur,
3264                                                          cur + iosize - 1,
3265                                                          NULL, 1);
3266                         else if (compressed) {
3267                                 /* we don't want to end_page_writeback on
3268                                  * a compressed extent.  this happens
3269                                  * elsewhere
3270                                  */
3271                                 nr++;
3272                         }
3273
3274                         cur += iosize;
3275                         pg_offset += iosize;
3276                         continue;
3277                 }
3278                 /* leave this out until we have a page_mkwrite call */
3279                 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
3280                                    EXTENT_DIRTY, 0, NULL)) {
3281                         cur = cur + iosize;
3282                         pg_offset += iosize;
3283                         continue;
3284                 }
3285
3286                 if (tree->ops && tree->ops->writepage_io_hook) {
3287                         ret = tree->ops->writepage_io_hook(page, cur,
3288                                                 cur + iosize - 1);
3289                 } else {
3290                         ret = 0;
3291                 }
3292                 if (ret) {
3293                         SetPageError(page);
3294                 } else {
3295                         unsigned long max_nr = end_index + 1;
3296
3297                         set_range_writeback(tree, cur, cur + iosize - 1);
3298                         if (!PageWriteback(page)) {
3299                                 printk(KERN_ERR "btrfs warning page %lu not "
3300                                        "writeback, cur %llu end %llu\n",
3301                                        page->index, cur, end);
3302                         }
3303
3304                         ret = submit_extent_page(write_flags, tree, page,
3305                                                  sector, iosize, pg_offset,
3306                                                  bdev, &epd->bio, max_nr,
3307                                                  end_bio_extent_writepage,
3308                                                  0, 0, 0);
3309                         if (ret)
3310                                 SetPageError(page);
3311                 }
3312                 cur = cur + iosize;
3313                 pg_offset += iosize;
3314                 nr++;
3315         }
3316 done:
3317         if (nr == 0) {
3318                 /* make sure the mapping tag for page dirty gets cleared */
3319                 set_page_writeback(page);
3320                 end_page_writeback(page);
3321         }
3322         unlock_page(page);
3323
3324 done_unlocked:
3325
3326         /* drop our reference on any cached states */
3327         free_extent_state(cached_state);
3328         return 0;
3329 }
3330
3331 static int eb_wait(void *word)
3332 {
3333         io_schedule();
3334         return 0;
3335 }
3336
3337 void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3338 {
3339         wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3340                     TASK_UNINTERRUPTIBLE);
3341 }
3342
3343 static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3344                                      struct btrfs_fs_info *fs_info,
3345                                      struct extent_page_data *epd)
3346 {
3347         unsigned long i, num_pages;
3348         int flush = 0;
3349         int ret = 0;
3350
3351         if (!btrfs_try_tree_write_lock(eb)) {
3352                 flush = 1;
3353                 flush_write_bio(epd);
3354                 btrfs_tree_lock(eb);
3355         }
3356
3357         if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3358                 btrfs_tree_unlock(eb);
3359                 if (!epd->sync_io)
3360                         return 0;
3361                 if (!flush) {
3362                         flush_write_bio(epd);
3363                         flush = 1;
3364                 }
3365                 while (1) {
3366                         wait_on_extent_buffer_writeback(eb);
3367                         btrfs_tree_lock(eb);
3368                         if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3369                                 break;
3370                         btrfs_tree_unlock(eb);
3371                 }
3372         }
3373
3374         /*
3375          * We need to do this to prevent races in people who check if the eb is
3376          * under IO since we can end up having no IO bits set for a short period
3377          * of time.
3378          */
3379         spin_lock(&eb->refs_lock);
3380         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3381                 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3382                 spin_unlock(&eb->refs_lock);
3383                 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3384                 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3385                                      -eb->len,
3386                                      fs_info->dirty_metadata_batch);
3387                 ret = 1;
3388         } else {
3389                 spin_unlock(&eb->refs_lock);
3390         }
3391
3392         btrfs_tree_unlock(eb);
3393
3394         if (!ret)
3395                 return ret;
3396
3397         num_pages = num_extent_pages(eb->start, eb->len);
3398         for (i = 0; i < num_pages; i++) {
3399                 struct page *p = extent_buffer_page(eb, i);
3400
3401                 if (!trylock_page(p)) {
3402                         if (!flush) {
3403                                 flush_write_bio(epd);
3404                                 flush = 1;
3405                         }
3406                         lock_page(p);
3407                 }
3408         }
3409
3410         return ret;
3411 }
3412
3413 static void end_extent_buffer_writeback(struct extent_buffer *eb)
3414 {
3415         clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3416         smp_mb__after_clear_bit();
3417         wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3418 }
3419
3420 static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3421 {
3422         int uptodate = err == 0;
3423         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3424         struct extent_buffer *eb;
3425         int done;
3426
3427         do {
3428                 struct page *page = bvec->bv_page;
3429
3430                 bvec--;
3431                 eb = (struct extent_buffer *)page->private;
3432                 BUG_ON(!eb);
3433                 done = atomic_dec_and_test(&eb->io_pages);
3434
3435                 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3436                         set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3437                         ClearPageUptodate(page);
3438                         SetPageError(page);
3439                 }
3440
3441                 end_page_writeback(page);
3442
3443                 if (!done)
3444                         continue;
3445
3446                 end_extent_buffer_writeback(eb);
3447         } while (bvec >= bio->bi_io_vec);
3448
3449         bio_put(bio);
3450
3451 }
3452
3453 static int write_one_eb(struct extent_buffer *eb,
3454                         struct btrfs_fs_info *fs_info,
3455                         struct writeback_control *wbc,
3456                         struct extent_page_data *epd)
3457 {
3458         struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3459         u64 offset = eb->start;
3460         unsigned long i, num_pages;
3461         unsigned long bio_flags = 0;
3462         int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
3463         int ret = 0;
3464
3465         clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3466         num_pages = num_extent_pages(eb->start, eb->len);
3467         atomic_set(&eb->io_pages, num_pages);
3468         if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3469                 bio_flags = EXTENT_BIO_TREE_LOG;
3470
3471         for (i = 0; i < num_pages; i++) {
3472                 struct page *p = extent_buffer_page(eb, i);
3473
3474                 clear_page_dirty_for_io(p);
3475                 set_page_writeback(p);
3476                 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3477                                          PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3478                                          -1, end_bio_extent_buffer_writepage,
3479                                          0, epd->bio_flags, bio_flags);
3480                 epd->bio_flags = bio_flags;
3481                 if (ret) {
3482                         set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3483                         SetPageError(p);
3484                         if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3485                                 end_extent_buffer_writeback(eb);
3486                         ret = -EIO;
3487                         break;
3488                 }
3489                 offset += PAGE_CACHE_SIZE;
3490                 update_nr_written(p, wbc, 1);
3491                 unlock_page(p);
3492         }
3493
3494         if (unlikely(ret)) {
3495                 for (; i < num_pages; i++) {
3496                         struct page *p = extent_buffer_page(eb, i);
3497                         unlock_page(p);
3498                 }
3499         }
3500
3501         return ret;
3502 }
3503
3504 int btree_write_cache_pages(struct address_space *mapping,
3505                                    struct writeback_control *wbc)
3506 {
3507         struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3508         struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3509         struct extent_buffer *eb, *prev_eb = NULL;
3510         struct extent_page_data epd = {
3511                 .bio = NULL,
3512                 .tree = tree,
3513                 .extent_locked = 0,
3514                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3515                 .bio_flags = 0,
3516         };
3517         int ret = 0;
3518         int done = 0;
3519         int nr_to_write_done = 0;
3520         struct pagevec pvec;
3521         int nr_pages;
3522         pgoff_t index;
3523         pgoff_t end;            /* Inclusive */
3524         int scanned = 0;
3525         int tag;
3526
3527         pagevec_init(&pvec, 0);
3528         if (wbc->range_cyclic) {
3529                 index = mapping->writeback_index; /* Start from prev offset */
3530                 end = -1;
3531         } else {
3532                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3533                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3534                 scanned = 1;
3535         }
3536         if (wbc->sync_mode == WB_SYNC_ALL)
3537                 tag = PAGECACHE_TAG_TOWRITE;
3538         else
3539                 tag = PAGECACHE_TAG_DIRTY;
3540 retry:
3541         if (wbc->sync_mode == WB_SYNC_ALL)
3542                 tag_pages_for_writeback(mapping, index, end);
3543         while (!done && !nr_to_write_done && (index <= end) &&
3544                (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3545                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3546                 unsigned i;
3547
3548                 scanned = 1;
3549                 for (i = 0; i < nr_pages; i++) {
3550                         struct page *page = pvec.pages[i];
3551
3552                         if (!PagePrivate(page))
3553                                 continue;
3554
3555                         if (!wbc->range_cyclic && page->index > end) {
3556                                 done = 1;
3557                                 break;
3558                         }
3559
3560                         spin_lock(&mapping->private_lock);
3561                         if (!PagePrivate(page)) {
3562                                 spin_unlock(&mapping->private_lock);
3563                                 continue;
3564                         }
3565
3566                         eb = (struct extent_buffer *)page->private;
3567
3568                         /*
3569                          * Shouldn't happen and normally this would be a BUG_ON
3570                          * but no sense in crashing the users box for something
3571                          * we can survive anyway.
3572                          */
3573                         if (!eb) {
3574                                 spin_unlock(&mapping->private_lock);
3575                                 WARN_ON(1);
3576                                 continue;
3577                         }
3578
3579                         if (eb == prev_eb) {
3580                                 spin_unlock(&mapping->private_lock);
3581                                 continue;
3582                         }
3583
3584                         ret = atomic_inc_not_zero(&eb->refs);
3585                         spin_unlock(&mapping->private_lock);
3586                         if (!ret)
3587                                 continue;
3588
3589                         prev_eb = eb;
3590                         ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3591                         if (!ret) {
3592                                 free_extent_buffer(eb);
3593                                 continue;
3594                         }
3595
3596                         ret = write_one_eb(eb, fs_info, wbc, &epd);
3597                         if (ret) {
3598                                 done = 1;
3599                                 free_extent_buffer(eb);
3600                                 break;
3601                         }
3602                         free_extent_buffer(eb);
3603
3604                         /*
3605                          * the filesystem may choose to bump up nr_to_write.
3606                          * We have to make sure to honor the new nr_to_write
3607                          * at any time
3608                          */
3609                         nr_to_write_done = wbc->nr_to_write <= 0;
3610                 }
3611                 pagevec_release(&pvec);
3612                 cond_resched();
3613         }
3614         if (!scanned && !done) {
3615                 /*
3616                  * We hit the last page and there is more work to be done: wrap
3617                  * back to the start of the file
3618                  */
3619                 scanned = 1;
3620                 index = 0;
3621                 goto retry;
3622         }
3623         flush_write_bio(&epd);
3624         return ret;
3625 }
3626
3627 /**
3628  * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3629  * @mapping: address space structure to write
3630  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3631  * @writepage: function called for each page
3632  * @data: data passed to writepage function
3633  *
3634  * If a page is already under I/O, write_cache_pages() skips it, even
3635  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
3636  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
3637  * and msync() need to guarantee that all the data which was dirty at the time
3638  * the call was made get new I/O started against them.  If wbc->sync_mode is
3639  * WB_SYNC_ALL then we were called for data integrity and we must wait for
3640  * existing IO to complete.
3641  */
3642 static int extent_write_cache_pages(struct extent_io_tree *tree,
3643                              struct address_space *mapping,
3644                              struct writeback_control *wbc,
3645                              writepage_t writepage, void *data,
3646                              void (*flush_fn)(void *))
3647 {
3648         struct inode *inode = mapping->host;
3649         int ret = 0;
3650         int done = 0;
3651         int nr_to_write_done = 0;
3652         struct pagevec pvec;
3653         int nr_pages;
3654         pgoff_t index;
3655         pgoff_t end;            /* Inclusive */
3656         int scanned = 0;
3657         int tag;
3658
3659         /*
3660          * We have to hold onto the inode so that ordered extents can do their
3661          * work when the IO finishes.  The alternative to this is failing to add
3662          * an ordered extent if the igrab() fails there and that is a huge pain
3663          * to deal with, so instead just hold onto the inode throughout the
3664          * writepages operation.  If it fails here we are freeing up the inode
3665          * anyway and we'd rather not waste our time writing out stuff that is
3666          * going to be truncated anyway.
3667          */
3668         if (!igrab(inode))
3669                 return 0;
3670
3671         pagevec_init(&pvec, 0);
3672         if (wbc->range_cyclic) {
3673                 index = mapping->writeback_index; /* Start from prev offset */
3674                 end = -1;
3675         } else {
3676                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3677                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3678                 scanned = 1;
3679         }
3680         if (wbc->sync_mode == WB_SYNC_ALL)
3681                 tag = PAGECACHE_TAG_TOWRITE;
3682         else
3683                 tag = PAGECACHE_TAG_DIRTY;
3684 retry:
3685         if (wbc->sync_mode == WB_SYNC_ALL)
3686                 tag_pages_for_writeback(mapping, index, end);
3687         while (!done && !nr_to_write_done && (index <= end) &&
3688                (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3689                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3690                 unsigned i;
3691
3692                 scanned = 1;
3693                 for (i = 0; i < nr_pages; i++) {
3694                         struct page *page = pvec.pages[i];
3695
3696                         /*
3697                          * At this point we hold neither mapping->tree_lock nor
3698                          * lock on the page itself: the page may be truncated or
3699                          * invalidated (changing page->mapping to NULL), or even
3700                          * swizzled back from swapper_space to tmpfs file
3701                          * mapping
3702                          */
3703                         if (!trylock_page(page)) {
3704                                 flush_fn(data);
3705                                 lock_page(page);
3706                         }
3707
3708                         if (unlikely(page->mapping != mapping)) {
3709                                 unlock_page(page);
3710                                 continue;
3711                         }
3712
3713                         if (!wbc->range_cyclic && page->index > end) {
3714                                 done = 1;
3715                                 unlock_page(page);
3716                                 continue;
3717                         }
3718
3719                         if (wbc->sync_mode != WB_SYNC_NONE) {
3720                                 if (PageWriteback(page))
3721                                         flush_fn(data);
3722                                 wait_on_page_writeback(page);
3723                         }
3724
3725                         if (PageWriteback(page) ||
3726                             !clear_page_dirty_for_io(page)) {
3727                                 unlock_page(page);
3728                                 continue;
3729                         }
3730
3731                         ret = (*writepage)(page, wbc, data);
3732
3733                         if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3734                                 unlock_page(page);
3735                                 ret = 0;
3736                         }
3737                         if (ret)
3738                                 done = 1;
3739
3740                         /*
3741                          * the filesystem may choose to bump up nr_to_write.
3742                          * We have to make sure to honor the new nr_to_write
3743                          * at any time
3744                          */
3745                         nr_to_write_done = wbc->nr_to_write <= 0;
3746                 }
3747                 pagevec_release(&pvec);
3748                 cond_resched();
3749         }
3750         if (!scanned && !done) {
3751                 /*
3752                  * We hit the last page and there is more work to be done: wrap
3753                  * back to the start of the file
3754                  */
3755                 scanned = 1;
3756                 index = 0;
3757                 goto retry;
3758         }
3759         btrfs_add_delayed_iput(inode);
3760         return ret;
3761 }
3762
3763 static void flush_epd_write_bio(struct extent_page_data *epd)
3764 {
3765         if (epd->bio) {
3766                 int rw = WRITE;
3767                 int ret;
3768
3769                 if (epd->sync_io)
3770                         rw = WRITE_SYNC;
3771
3772                 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
3773                 BUG_ON(ret < 0); /* -ENOMEM */
3774                 epd->bio = NULL;
3775         }
3776 }
3777
3778 static noinline void flush_write_bio(void *data)
3779 {
3780         struct extent_page_data *epd = data;
3781         flush_epd_write_bio(epd);
3782 }
3783
3784 int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3785                           get_extent_t *get_extent,
3786                           struct writeback_control *wbc)
3787 {
3788         int ret;
3789         struct extent_page_data epd = {
3790                 .bio = NULL,
3791                 .tree = tree,
3792                 .get_extent = get_extent,
3793                 .extent_locked = 0,
3794                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3795                 .bio_flags = 0,
3796         };
3797
3798         ret = __extent_writepage(page, wbc, &epd);
3799
3800         flush_epd_write_bio(&epd);
3801         return ret;
3802 }
3803
3804 int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3805                               u64 start, u64 end, get_extent_t *get_extent,
3806                               int mode)
3807 {
3808         int ret = 0;
3809         struct address_space *mapping = inode->i_mapping;
3810         struct page *page;
3811         unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3812                 PAGE_CACHE_SHIFT;
3813
3814         struct extent_page_data epd = {
3815                 .bio = NULL,
3816                 .tree = tree,
3817                 .get_extent = get_extent,
3818                 .extent_locked = 1,
3819                 .sync_io = mode == WB_SYNC_ALL,
3820                 .bio_flags = 0,
3821         };
3822         struct writeback_control wbc_writepages = {
3823                 .sync_mode      = mode,
3824                 .nr_to_write    = nr_pages * 2,
3825                 .range_start    = start,
3826                 .range_end      = end + 1,
3827         };
3828
3829         while (start <= end) {
3830                 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3831                 if (clear_page_dirty_for_io(page))
3832                         ret = __extent_writepage(page, &wbc_writepages, &epd);
3833                 else {
3834                         if (tree->ops && tree->ops->writepage_end_io_hook)
3835                                 tree->ops->writepage_end_io_hook(page, start,
3836                                                  start + PAGE_CACHE_SIZE - 1,
3837                                                  NULL, 1);
3838                         unlock_page(page);
3839                 }
3840                 page_cache_release(page);
3841                 start += PAGE_CACHE_SIZE;
3842         }
3843
3844         flush_epd_write_bio(&epd);
3845         return ret;
3846 }
3847
3848 int extent_writepages(struct extent_io_tree *tree,
3849                       struct address_space *mapping,
3850                       get_extent_t *get_extent,
3851                       struct writeback_control *wbc)
3852 {
3853         int ret = 0;
3854         struct extent_page_data epd = {
3855                 .bio = NULL,
3856                 .tree = tree,
3857                 .get_extent = get_extent,
3858                 .extent_locked = 0,
3859                 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3860                 .bio_flags = 0,
3861         };
3862
3863         ret = extent_write_cache_pages(tree, mapping, wbc,
3864                                        __extent_writepage, &epd,
3865                                        flush_write_bio);
3866         flush_epd_write_bio(&epd);
3867         return ret;
3868 }
3869
3870 int extent_readpages(struct extent_io_tree *tree,
3871                      struct address_space *mapping,
3872                      struct list_head *pages, unsigned nr_pages,
3873                      get_extent_t get_extent)
3874 {
3875         struct bio *bio = NULL;
3876         unsigned page_idx;
3877         unsigned long bio_flags = 0;
3878         struct page *pagepool[16];
3879         struct page *page;
3880         struct extent_map *em_cached = NULL;
3881         int nr = 0;
3882
3883         for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3884                 page = list_entry(pages->prev, struct page, lru);
3885
3886                 prefetchw(&page->flags);
3887                 list_del(&page->lru);
3888                 if (add_to_page_cache_lru(page, mapping,
3889                                         page->index, GFP_NOFS)) {
3890                         page_cache_release(page);
3891                         continue;
3892                 }
3893
3894                 pagepool[nr++] = page;
3895                 if (nr < ARRAY_SIZE(pagepool))
3896                         continue;
3897                 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3898                                    &bio, 0, &bio_flags, READ);
3899                 nr = 0;
3900         }
3901         if (nr)
3902                 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3903                                    &bio, 0, &bio_flags, READ);
3904
3905         if (em_cached)
3906                 free_extent_map(em_cached);
3907
3908         BUG_ON(!list_empty(pages));
3909         if (bio)
3910                 return submit_one_bio(READ, bio, 0, bio_flags);
3911         return 0;
3912 }
3913
3914 /*
3915  * basic invalidatepage code, this waits on any locked or writeback
3916  * ranges corresponding to the page, and then deletes any extent state
3917  * records from the tree
3918  */
3919 int extent_invalidatepage(struct extent_io_tree *tree,
3920                           struct page *page, unsigned long offset)
3921 {
3922         struct extent_state *cached_state = NULL;
3923         u64 start = page_offset(page);
3924         u64 end = start + PAGE_CACHE_SIZE - 1;
3925         size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3926
3927         start += ALIGN(offset, blocksize);
3928         if (start > end)
3929                 return 0;
3930
3931         lock_extent_bits(tree, start, end, 0, &cached_state);
3932         wait_on_page_writeback(page);
3933         clear_extent_bit(tree, start, end,
3934                          EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3935                          EXTENT_DO_ACCOUNTING,
3936                          1, 1, &cached_state, GFP_NOFS);
3937         return 0;
3938 }
3939
3940 /*
3941  * a helper for releasepage, this tests for areas of the page that
3942  * are locked or under IO and drops the related state bits if it is safe
3943  * to drop the page.
3944  */
3945 static int try_release_extent_state(struct extent_map_tree *map,
3946                                     struct extent_io_tree *tree,
3947                                     struct page *page, gfp_t mask)
3948 {
3949         u64 start = page_offset(page);
3950         u64 end = start + PAGE_CACHE_SIZE - 1;
3951         int ret = 1;
3952
3953         if (test_range_bit(tree, start, end,
3954                            EXTENT_IOBITS, 0, NULL))
3955                 ret = 0;
3956         else {
3957                 if ((mask & GFP_NOFS) == GFP_NOFS)
3958                         mask = GFP_NOFS;
3959                 /*
3960                  * at this point we can safely clear everything except the
3961                  * locked bit and the nodatasum bit
3962                  */
3963                 ret = clear_extent_bit(tree, start, end,
3964                                  ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3965                                  0, 0, NULL, mask);
3966
3967                 /* if clear_extent_bit failed for enomem reasons,
3968                  * we can't allow the release to continue.
3969                  */
3970                 if (ret < 0)
3971                         ret = 0;
3972                 else
3973                         ret = 1;
3974         }
3975         return ret;
3976 }
3977
3978 /*
3979  * a helper for releasepage.  As long as there are no locked extents
3980  * in the range corresponding to the page, both state records and extent
3981  * map records are removed
3982  */
3983 int try_release_extent_mapping(struct extent_map_tree *map,
3984                                struct extent_io_tree *tree, struct page *page,
3985                                gfp_t mask)
3986 {
3987         struct extent_map *em;
3988         u64 start = page_offset(page);
3989         u64 end = start + PAGE_CACHE_SIZE - 1;
3990
3991         if ((mask & __GFP_WAIT) &&
3992             page->mapping->host->i_size > 16 * 1024 * 1024) {
3993                 u64 len;
3994                 while (start <= end) {
3995                         len = end - start + 1;
3996                         write_lock(&map->lock);
3997                         em = lookup_extent_mapping(map, start, len);
3998                         if (!em) {
3999                                 write_unlock(&map->lock);
4000                                 break;
4001                         }
4002                         if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4003                             em->start != start) {
4004                                 write_unlock(&map->lock);
4005                                 free_extent_map(em);
4006                                 break;
4007                         }
4008                         if (!test_range_bit(tree, em->start,
4009                                             extent_map_end(em) - 1,
4010                                             EXTENT_LOCKED | EXTENT_WRITEBACK,
4011                                             0, NULL)) {
4012                                 remove_extent_mapping(map, em);
4013                                 /* once for the rb tree */
4014                                 free_extent_map(em);
4015                         }
4016                         start = extent_map_end(em);
4017                         write_unlock(&map->lock);
4018
4019                         /* once for us */
4020                         free_extent_map(em);
4021                 }
4022         }
4023         return try_release_extent_state(map, tree, page, mask);
4024 }
4025
4026 /*
4027  * helper function for fiemap, which doesn't want to see any holes.
4028  * This maps until we find something past 'last'
4029  */
4030 static struct extent_map *get_extent_skip_holes(struct inode *inode,
4031                                                 u64 offset,
4032                                                 u64 last,
4033                                                 get_extent_t *get_extent)
4034 {
4035         u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4036         struct extent_map *em;
4037         u64 len;
4038
4039         if (offset >= last)
4040                 return NULL;
4041
4042         while(1) {
4043                 len = last - offset;
4044                 if (len == 0)
4045                         break;
4046                 len = ALIGN(len, sectorsize);
4047                 em = get_extent(inode, NULL, 0, offset, len, 0);
4048                 if (IS_ERR_OR_NULL(em))
4049                         return em;
4050
4051                 /* if this isn't a hole return it */
4052                 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4053                     em->block_start != EXTENT_MAP_HOLE) {
4054                         return em;
4055                 }
4056
4057                 /* this is a hole, advance to the next extent */
4058                 offset = extent_map_end(em);
4059                 free_extent_map(em);
4060                 if (offset >= last)
4061                         break;
4062         }
4063         return NULL;
4064 }
4065
4066 static noinline int count_ext_ref(u64 inum, u64 offset, u64 root_id, void *ctx)
4067 {
4068         unsigned long cnt = *((unsigned long *)ctx);
4069
4070         cnt++;
4071         *((unsigned long *)ctx) = cnt;
4072
4073         /* Now we're sure that the extent is shared. */
4074         if (cnt > 1)
4075                 return 1;
4076         return 0;
4077 }
4078
4079 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4080                 __u64 start, __u64 len, get_extent_t *get_extent)
4081 {
4082         int ret = 0;
4083         u64 off = start;
4084         u64 max = start + len;
4085         u32 flags = 0;
4086         u32 found_type;
4087         u64 last;
4088         u64 last_for_get_extent = 0;
4089         u64 disko = 0;
4090         u64 isize = i_size_read(inode);
4091         struct btrfs_key found_key;
4092         struct extent_map *em = NULL;
4093         struct extent_state *cached_state = NULL;
4094         struct btrfs_path *path;
4095         struct btrfs_file_extent_item *item;
4096         int end = 0;
4097         u64 em_start = 0;
4098         u64 em_len = 0;
4099         u64 em_end = 0;
4100         unsigned long emflags;
4101
4102         if (len == 0)
4103                 return -EINVAL;
4104
4105         path = btrfs_alloc_path();
4106         if (!path)
4107                 return -ENOMEM;
4108         path->leave_spinning = 1;
4109
4110         start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
4111         len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
4112
4113         /*
4114          * lookup the last file extent.  We're not using i_size here
4115          * because there might be preallocation past i_size
4116          */
4117         ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
4118                                        path, btrfs_ino(inode), -1, 0);
4119         if (ret < 0) {
4120                 btrfs_free_path(path);
4121                 return ret;
4122         }
4123         WARN_ON(!ret);
4124         path->slots[0]--;
4125         item = btrfs_item_ptr(path->nodes[0], path->slots[0],
4126                               struct btrfs_file_extent_item);
4127         btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4128         found_type = btrfs_key_type(&found_key);
4129
4130         /* No extents, but there might be delalloc bits */
4131         if (found_key.objectid != btrfs_ino(inode) ||
4132             found_type != BTRFS_EXTENT_DATA_KEY) {
4133                 /* have to trust i_size as the end */
4134                 last = (u64)-1;
4135                 last_for_get_extent = isize;
4136         } else {
4137                 /*
4138                  * remember the start of the last extent.  There are a
4139                  * bunch of different factors that go into the length of the
4140                  * extent, so its much less complex to remember where it started
4141                  */
4142                 last = found_key.offset;
4143                 last_for_get_extent = last + 1;
4144         }
4145         btrfs_release_path(path);
4146
4147         /*
4148          * we might have some extents allocated but more delalloc past those
4149          * extents.  so, we trust isize unless the start of the last extent is
4150          * beyond isize
4151          */
4152         if (last < isize) {
4153                 last = (u64)-1;
4154                 last_for_get_extent = isize;
4155         }
4156
4157         lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
4158                          &cached_state);
4159
4160         em = get_extent_skip_holes(inode, start, last_for_get_extent,
4161                                    get_extent);
4162         if (!em)
4163                 goto out;
4164         if (IS_ERR(em)) {
4165                 ret = PTR_ERR(em);
4166                 goto out;
4167         }
4168
4169         while (!end) {
4170                 u64 offset_in_extent = 0;
4171
4172                 /* break if the extent we found is outside the range */
4173                 if (em->start >= max || extent_map_end(em) < off)
4174                         break;
4175
4176                 /*
4177                  * get_extent may return an extent that starts before our
4178                  * requested range.  We have to make sure the ranges
4179                  * we return to fiemap always move forward and don't
4180                  * overlap, so adjust the offsets here
4181                  */
4182                 em_start = max(em->start, off);
4183
4184                 /*
4185                  * record the offset from the start of the extent
4186                  * for adjusting the disk offset below.  Only do this if the
4187                  * extent isn't compressed since our in ram offset may be past
4188                  * what we have actually allocated on disk.
4189                  */
4190                 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4191                         offset_in_extent = em_start - em->start;
4192                 em_end = extent_map_end(em);
4193                 em_len = em_end - em_start;
4194                 emflags = em->flags;
4195                 disko = 0;
4196                 flags = 0;
4197
4198                 /*
4199                  * bump off for our next call to get_extent
4200                  */
4201                 off = extent_map_end(em);
4202                 if (off >= max)
4203                         end = 1;
4204
4205                 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
4206                         end = 1;
4207                         flags |= FIEMAP_EXTENT_LAST;
4208                 } else if (em->block_start == EXTENT_MAP_INLINE) {
4209                         flags |= (FIEMAP_EXTENT_DATA_INLINE |
4210                                   FIEMAP_EXTENT_NOT_ALIGNED);
4211                 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
4212                         flags |= (FIEMAP_EXTENT_DELALLOC |
4213                                   FIEMAP_EXTENT_UNKNOWN);
4214                 } else {
4215                         unsigned long ref_cnt = 0;
4216
4217                         disko = em->block_start + offset_in_extent;
4218
4219                         /*
4220                          * As btrfs supports shared space, this information
4221                          * can be exported to userspace tools via
4222                          * flag FIEMAP_EXTENT_SHARED.
4223                          */
4224                         ret = iterate_inodes_from_logical(
4225                                         em->block_start,
4226                                         BTRFS_I(inode)->root->fs_info,
4227                                         path, count_ext_ref, &ref_cnt);
4228                         if (ret < 0 && ret != -ENOENT)
4229                                 goto out_free;
4230
4231                         if (ref_cnt > 1)
4232                                 flags |= FIEMAP_EXTENT_SHARED;
4233                 }
4234                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4235                         flags |= FIEMAP_EXTENT_ENCODED;
4236
4237                 free_extent_map(em);
4238                 em = NULL;
4239                 if ((em_start >= last) || em_len == (u64)-1 ||
4240                    (last == (u64)-1 && isize <= em_end)) {
4241                         flags |= FIEMAP_EXTENT_LAST;
4242                         end = 1;
4243                 }
4244
4245                 /* now scan forward to see if this is really the last extent. */
4246                 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4247                                            get_extent);
4248                 if (IS_ERR(em)) {
4249                         ret = PTR_ERR(em);
4250                         goto out;
4251                 }
4252                 if (!em) {
4253                         flags |= FIEMAP_EXTENT_LAST;
4254                         end = 1;
4255                 }
4256                 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4257                                               em_len, flags);
4258                 if (ret)
4259                         goto out_free;
4260         }
4261 out_free:
4262         free_extent_map(em);
4263 out:
4264         btrfs_free_path(path);
4265         unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
4266                              &cached_state, GFP_NOFS);
4267         return ret;
4268 }
4269
4270 static void __free_extent_buffer(struct extent_buffer *eb)
4271 {
4272         btrfs_leak_debug_del(&eb->leak_list);
4273         kmem_cache_free(extent_buffer_cache, eb);
4274 }
4275
4276 static int extent_buffer_under_io(struct extent_buffer *eb)
4277 {
4278         return (atomic_read(&eb->io_pages) ||
4279                 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4280                 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4281 }
4282
4283 /*
4284  * Helper for releasing extent buffer page.
4285  */
4286 static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4287                                                 unsigned long start_idx)
4288 {
4289         unsigned long index;
4290         unsigned long num_pages;
4291         struct page *page;
4292         int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4293
4294         BUG_ON(extent_buffer_under_io(eb));
4295
4296         num_pages = num_extent_pages(eb->start, eb->len);
4297         index = start_idx + num_pages;
4298         if (start_idx >= index)
4299                 return;
4300
4301         do {
4302                 index--;
4303                 page = extent_buffer_page(eb, index);
4304                 if (page && mapped) {
4305                         spin_lock(&page->mapping->private_lock);
4306                         /*
4307                          * We do this since we'll remove the pages after we've
4308                          * removed the eb from the radix tree, so we could race
4309                          * and have this page now attached to the new eb.  So
4310                          * only clear page_private if it's still connected to
4311                          * this eb.
4312                          */
4313                         if (PagePrivate(page) &&
4314                             page->private == (unsigned long)eb) {
4315                                 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4316                                 BUG_ON(PageDirty(page));
4317                                 BUG_ON(PageWriteback(page));
4318                                 /*
4319                                  * We need to make sure we haven't be attached
4320                                  * to a new eb.
4321                                  */
4322                                 ClearPagePrivate(page);
4323                                 set_page_private(page, 0);
4324                                 /* One for the page private */
4325                                 page_cache_release(page);
4326                         }
4327                         spin_unlock(&page->mapping->private_lock);
4328
4329                 }
4330                 if (page) {
4331                         /* One for when we alloced the page */
4332                         page_cache_release(page);
4333                 }
4334         } while (index != start_idx);
4335 }
4336
4337 /*
4338  * Helper for releasing the extent buffer.
4339  */
4340 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4341 {
4342         btrfs_release_extent_buffer_page(eb, 0);
4343         __free_extent_buffer(eb);
4344 }
4345
4346 static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
4347                                                    u64 start,
4348                                                    unsigned long len,
4349                                                    gfp_t mask)
4350 {
4351         struct extent_buffer *eb = NULL;
4352
4353         eb = kmem_cache_zalloc(extent_buffer_cache, mask);
4354         if (eb == NULL)
4355                 return NULL;
4356         eb->start = start;
4357         eb->len = len;
4358         eb->tree = tree;
4359         eb->bflags = 0;
4360         rwlock_init(&eb->lock);
4361         atomic_set(&eb->write_locks, 0);
4362         atomic_set(&eb->read_locks, 0);
4363         atomic_set(&eb->blocking_readers, 0);
4364         atomic_set(&eb->blocking_writers, 0);
4365         atomic_set(&eb->spinning_readers, 0);
4366         atomic_set(&eb->spinning_writers, 0);
4367         eb->lock_nested = 0;
4368         init_waitqueue_head(&eb->write_lock_wq);
4369         init_waitqueue_head(&eb->read_lock_wq);
4370
4371         btrfs_leak_debug_add(&eb->leak_list, &buffers);
4372
4373         spin_lock_init(&eb->refs_lock);
4374         atomic_set(&eb->refs, 1);
4375         atomic_set(&eb->io_pages, 0);
4376
4377         /*
4378          * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4379          */
4380         BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4381                 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4382         BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4383
4384         return eb;
4385 }
4386
4387 struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4388 {
4389         unsigned long i;
4390         struct page *p;
4391         struct extent_buffer *new;
4392         unsigned long num_pages = num_extent_pages(src->start, src->len);
4393
4394         new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS);
4395         if (new == NULL)
4396                 return NULL;
4397
4398         for (i = 0; i < num_pages; i++) {
4399                 p = alloc_page(GFP_NOFS);
4400                 if (!p) {
4401                         btrfs_release_extent_buffer(new);
4402                         return NULL;
4403                 }
4404                 attach_extent_buffer_page(new, p);
4405                 WARN_ON(PageDirty(p));
4406                 SetPageUptodate(p);
4407                 new->pages[i] = p;
4408         }
4409
4410         copy_extent_buffer(new, src, 0, 0, src->len);
4411         set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4412         set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4413
4414         return new;
4415 }
4416
4417 struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4418 {
4419         struct extent_buffer *eb;
4420         unsigned long num_pages = num_extent_pages(0, len);
4421         unsigned long i;
4422
4423         eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS);
4424         if (!eb)
4425                 return NULL;
4426
4427         for (i = 0; i < num_pages; i++) {
4428                 eb->pages[i] = alloc_page(GFP_NOFS);
4429                 if (!eb->pages[i])
4430                         goto err;
4431         }
4432         set_extent_buffer_uptodate(eb);
4433         btrfs_set_header_nritems(eb, 0);
4434         set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4435
4436         return eb;
4437 err:
4438         for (; i > 0; i--)
4439                 __free_page(eb->pages[i - 1]);
4440         __free_extent_buffer(eb);
4441         return NULL;
4442 }
4443
4444 static void check_buffer_tree_ref(struct extent_buffer *eb)
4445 {
4446         int refs;
4447         /* the ref bit is tricky.  We have to make sure it is set
4448          * if we have the buffer dirty.   Otherwise the
4449          * code to free a buffer can end up dropping a dirty
4450          * page
4451          *
4452          * Once the ref bit is set, it won't go away while the
4453          * buffer is dirty or in writeback, and it also won't
4454          * go away while we have the reference count on the
4455          * eb bumped.
4456          *
4457          * We can't just set the ref bit without bumping the
4458          * ref on the eb because free_extent_buffer might
4459          * see the ref bit and try to clear it.  If this happens
4460          * free_extent_buffer might end up dropping our original
4461          * ref by mistake and freeing the page before we are able
4462          * to add one more ref.
4463          *
4464          * So bump the ref count first, then set the bit.  If someone
4465          * beat us to it, drop the ref we added.
4466          */
4467         refs = atomic_read(&eb->refs);
4468         if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4469                 return;
4470
4471         spin_lock(&eb->refs_lock);
4472         if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4473                 atomic_inc(&eb->refs);
4474         spin_unlock(&eb->refs_lock);
4475 }
4476
4477 static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4478 {
4479         unsigned long num_pages, i;
4480
4481         check_buffer_tree_ref(eb);
4482
4483         num_pages = num_extent_pages(eb->start, eb->len);
4484         for (i = 0; i < num_pages; i++) {
4485                 struct page *p = extent_buffer_page(eb, i);
4486                 mark_page_accessed(p);
4487         }
4488 }
4489
4490 struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
4491                                           u64 start, unsigned long len)
4492 {
4493         unsigned long num_pages = num_extent_pages(start, len);
4494         unsigned long i;
4495         unsigned long index = start >> PAGE_CACHE_SHIFT;
4496         struct extent_buffer *eb;
4497         struct extent_buffer *exists = NULL;
4498         struct page *p;
4499         struct address_space *mapping = tree->mapping;
4500         int uptodate = 1;
4501         int ret;
4502
4503         rcu_read_lock();
4504         eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4505         if (eb && atomic_inc_not_zero(&eb->refs)) {
4506                 rcu_read_unlock();
4507                 mark_extent_buffer_accessed(eb);
4508                 return eb;
4509         }
4510         rcu_read_unlock();
4511
4512         eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
4513         if (!eb)
4514                 return NULL;
4515
4516         for (i = 0; i < num_pages; i++, index++) {
4517                 p = find_or_create_page(mapping, index, GFP_NOFS);
4518                 if (!p)
4519                         goto free_eb;
4520
4521                 spin_lock(&mapping->private_lock);
4522                 if (PagePrivate(p)) {
4523                         /*
4524                          * We could have already allocated an eb for this page
4525                          * and attached one so lets see if we can get a ref on
4526                          * the existing eb, and if we can we know it's good and
4527                          * we can just return that one, else we know we can just
4528                          * overwrite page->private.
4529                          */
4530                         exists = (struct extent_buffer *)p->private;
4531                         if (atomic_inc_not_zero(&exists->refs)) {
4532                                 spin_unlock(&mapping->private_lock);
4533                                 unlock_page(p);
4534                                 page_cache_release(p);
4535                                 mark_extent_buffer_accessed(exists);
4536                                 goto free_eb;
4537                         }
4538
4539                         /*
4540                          * Do this so attach doesn't complain and we need to
4541                          * drop the ref the old guy had.
4542                          */
4543                         ClearPagePrivate(p);
4544                         WARN_ON(PageDirty(p));
4545                         page_cache_release(p);
4546                 }
4547                 attach_extent_buffer_page(eb, p);
4548                 spin_unlock(&mapping->private_lock);
4549                 WARN_ON(PageDirty(p));
4550                 mark_page_accessed(p);
4551                 eb->pages[i] = p;
4552                 if (!PageUptodate(p))
4553                         uptodate = 0;
4554
4555                 /*
4556                  * see below about how we avoid a nasty race with release page
4557                  * and why we unlock later
4558                  */
4559         }
4560         if (uptodate)
4561                 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4562 again:
4563         ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4564         if (ret)
4565                 goto free_eb;
4566
4567         spin_lock(&tree->buffer_lock);
4568         ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4569         if (ret == -EEXIST) {
4570                 exists = radix_tree_lookup(&tree->buffer,
4571                                                 start >> PAGE_CACHE_SHIFT);
4572                 if (!atomic_inc_not_zero(&exists->refs)) {
4573                         spin_unlock(&tree->buffer_lock);
4574                         radix_tree_preload_end();
4575                         exists = NULL;
4576                         goto again;
4577                 }
4578                 spin_unlock(&tree->buffer_lock);
4579                 radix_tree_preload_end();
4580                 mark_extent_buffer_accessed(exists);
4581                 goto free_eb;
4582         }
4583         /* add one reference for the tree */
4584         check_buffer_tree_ref(eb);
4585         spin_unlock(&tree->buffer_lock);
4586         radix_tree_preload_end();
4587
4588         /*
4589          * there is a race where release page may have
4590          * tried to find this extent buffer in the radix
4591          * but failed.  It will tell the VM it is safe to
4592          * reclaim the, and it will clear the page private bit.
4593          * We must make sure to set the page private bit properly
4594          * after the extent buffer is in the radix tree so
4595          * it doesn't get lost
4596          */
4597         SetPageChecked(eb->pages[0]);
4598         for (i = 1; i < num_pages; i++) {
4599                 p = extent_buffer_page(eb, i);
4600                 ClearPageChecked(p);
4601                 unlock_page(p);
4602         }
4603         unlock_page(eb->pages[0]);
4604         return eb;
4605
4606 free_eb:
4607         for (i = 0; i < num_pages; i++) {
4608                 if (eb->pages[i])
4609                         unlock_page(eb->pages[i]);
4610         }
4611
4612         WARN_ON(!atomic_dec_and_test(&eb->refs));
4613         btrfs_release_extent_buffer(eb);
4614         return exists;
4615 }
4616
4617 struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
4618                                          u64 start, unsigned long len)
4619 {
4620         struct extent_buffer *eb;
4621
4622         rcu_read_lock();
4623         eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4624         if (eb && atomic_inc_not_zero(&eb->refs)) {
4625                 rcu_read_unlock();
4626                 mark_extent_buffer_accessed(eb);
4627                 return eb;
4628         }
4629         rcu_read_unlock();
4630
4631         return NULL;
4632 }
4633
4634 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4635 {
4636         struct extent_buffer *eb =
4637                         container_of(head, struct extent_buffer, rcu_head);
4638
4639         __free_extent_buffer(eb);
4640 }
4641
4642 /* Expects to have eb->eb_lock already held */
4643 static int release_extent_buffer(struct extent_buffer *eb)
4644 {
4645         WARN_ON(atomic_read(&eb->refs) == 0);
4646         if (atomic_dec_and_test(&eb->refs)) {
4647                 if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
4648                         spin_unlock(&eb->refs_lock);
4649                 } else {
4650                         struct extent_io_tree *tree = eb->tree;
4651
4652                         spin_unlock(&eb->refs_lock);
4653
4654                         spin_lock(&tree->buffer_lock);
4655                         radix_tree_delete(&tree->buffer,
4656                                           eb->start >> PAGE_CACHE_SHIFT);
4657                         spin_unlock(&tree->buffer_lock);
4658                 }
4659
4660                 /* Should be safe to release our pages at this point */
4661                 btrfs_release_extent_buffer_page(eb, 0);
4662                 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4663                 return 1;
4664         }
4665         spin_unlock(&eb->refs_lock);
4666
4667         return 0;
4668 }
4669
4670 void free_extent_buffer(struct extent_buffer *eb)
4671 {
4672         int refs;
4673         int old;
4674         if (!eb)
4675                 return;
4676
4677         while (1) {
4678                 refs = atomic_read(&eb->refs);
4679                 if (refs <= 3)
4680                         break;
4681                 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4682                 if (old == refs)
4683                         return;
4684         }
4685
4686         spin_lock(&eb->refs_lock);
4687         if (atomic_read(&eb->refs) == 2 &&
4688             test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4689                 atomic_dec(&eb->refs);
4690
4691         if (atomic_read(&eb->refs) == 2 &&
4692             test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4693             !extent_buffer_under_io(eb) &&
4694             test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4695                 atomic_dec(&eb->refs);
4696
4697         /*
4698          * I know this is terrible, but it's temporary until we stop tracking
4699          * the uptodate bits and such for the extent buffers.
4700          */
4701         release_extent_buffer(eb);
4702 }
4703
4704 void free_extent_buffer_stale(struct extent_buffer *eb)
4705 {
4706         if (!eb)
4707                 return;
4708
4709         spin_lock(&eb->refs_lock);
4710         set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4711
4712         if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4713             test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4714                 atomic_dec(&eb->refs);
4715         release_extent_buffer(eb);
4716 }
4717
4718 void clear_extent_buffer_dirty(struct extent_buffer *eb)
4719 {
4720         unsigned long i;
4721         unsigned long num_pages;
4722         struct page *page;
4723
4724         num_pages = num_extent_pages(eb->start, eb->len);
4725
4726         for (i = 0; i < num_pages; i++) {
4727                 page = extent_buffer_page(eb, i);
4728                 if (!PageDirty(page))
4729                         continue;
4730
4731                 lock_page(page);
4732                 WARN_ON(!PagePrivate(page));
4733
4734                 clear_page_dirty_for_io(page);
4735                 spin_lock_irq(&page->mapping->tree_lock);
4736                 if (!PageDirty(page)) {
4737                         radix_tree_tag_clear(&page->mapping->page_tree,
4738                                                 page_index(page),
4739                                                 PAGECACHE_TAG_DIRTY);
4740                 }
4741                 spin_unlock_irq(&page->mapping->tree_lock);
4742                 ClearPageError(page);
4743                 unlock_page(page);
4744         }
4745         WARN_ON(atomic_read(&eb->refs) == 0);
4746 }
4747
4748 int set_extent_buffer_dirty(struct extent_buffer *eb)
4749 {
4750         unsigned long i;
4751         unsigned long num_pages;
4752         int was_dirty = 0;
4753
4754         check_buffer_tree_ref(eb);
4755
4756         was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
4757
4758         num_pages = num_extent_pages(eb->start, eb->len);
4759         WARN_ON(atomic_read(&eb->refs) == 0);
4760         WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4761
4762         for (i = 0; i < num_pages; i++)
4763                 set_page_dirty(extent_buffer_page(eb, i));
4764         return was_dirty;
4765 }
4766
4767 int clear_extent_buffer_uptodate(struct extent_buffer *eb)
4768 {
4769         unsigned long i;
4770         struct page *page;
4771         unsigned long num_pages;
4772
4773         clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4774         num_pages = num_extent_pages(eb->start, eb->len);
4775         for (i = 0; i < num_pages; i++) {
4776                 page = extent_buffer_page(eb, i);
4777                 if (page)
4778                         ClearPageUptodate(page);
4779         }
4780         return 0;
4781 }
4782
4783 int set_extent_buffer_uptodate(struct extent_buffer *eb)
4784 {
4785         unsigned long i;
4786         struct page *page;
4787         unsigned long num_pages;
4788
4789         set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4790         num_pages = num_extent_pages(eb->start, eb->len);
4791         for (i = 0; i < num_pages; i++) {
4792                 page = extent_buffer_page(eb, i);
4793                 SetPageUptodate(page);
4794         }
4795         return 0;
4796 }
4797
4798 int extent_buffer_uptodate(struct extent_buffer *eb)
4799 {
4800         return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4801 }
4802
4803 int read_extent_buffer_pages(struct extent_io_tree *tree,
4804                              struct extent_buffer *eb, u64 start, int wait,
4805                              get_extent_t *get_extent, int mirror_num)
4806 {
4807         unsigned long i;
4808         unsigned long start_i;
4809         struct page *page;
4810         int err;
4811         int ret = 0;
4812         int locked_pages = 0;
4813         int all_uptodate = 1;
4814         unsigned long num_pages;
4815         unsigned long num_reads = 0;
4816         struct bio *bio = NULL;
4817         unsigned long bio_flags = 0;
4818
4819         if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4820                 return 0;
4821
4822         if (start) {
4823                 WARN_ON(start < eb->start);
4824                 start_i = (start >> PAGE_CACHE_SHIFT) -
4825                         (eb->start >> PAGE_CACHE_SHIFT);
4826         } else {
4827                 start_i = 0;
4828         }
4829
4830         num_pages = num_extent_pages(eb->start, eb->len);
4831         for (i = start_i; i < num_pages; i++) {
4832                 page = extent_buffer_page(eb, i);
4833                 if (wait == WAIT_NONE) {
4834                         if (!trylock_page(page))
4835                                 goto unlock_exit;
4836                 } else {
4837                         lock_page(page);
4838                 }
4839                 locked_pages++;
4840                 if (!PageUptodate(page)) {
4841                         num_reads++;
4842                         all_uptodate = 0;
4843                 }
4844         }
4845         if (all_uptodate) {
4846                 if (start_i == 0)
4847                         set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4848                 goto unlock_exit;
4849         }
4850
4851         clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
4852         eb->read_mirror = 0;
4853         atomic_set(&eb->io_pages, num_reads);
4854         for (i = start_i; i < num_pages; i++) {
4855                 page = extent_buffer_page(eb, i);
4856                 if (!PageUptodate(page)) {
4857                         ClearPageError(page);
4858                         err = __extent_read_full_page(tree, page,
4859                                                       get_extent, &bio,
4860                                                       mirror_num, &bio_flags,
4861                                                       READ | REQ_META);
4862                         if (err)
4863                                 ret = err;
4864                 } else {
4865                         unlock_page(page);
4866                 }
4867         }
4868
4869         if (bio) {
4870                 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
4871                                      bio_flags);
4872                 if (err)
4873                         return err;
4874         }
4875
4876         if (ret || wait != WAIT_COMPLETE)
4877                 return ret;
4878
4879         for (i = start_i; i < num_pages; i++) {
4880                 page = extent_buffer_page(eb, i);
4881                 wait_on_page_locked(page);
4882                 if (!PageUptodate(page))
4883                         ret = -EIO;
4884         }
4885
4886         return ret;
4887
4888 unlock_exit:
4889         i = start_i;
4890         while (locked_pages > 0) {
4891                 page = extent_buffer_page(eb, i);
4892                 i++;
4893                 unlock_page(page);
4894                 locked_pages--;
4895         }
4896         return ret;
4897 }
4898
4899 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4900                         unsigned long start,
4901                         unsigned long len)
4902 {
4903         size_t cur;
4904         size_t offset;
4905         struct page *page;
4906         char *kaddr;
4907         char *dst = (char *)dstv;
4908         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4909         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4910
4911         WARN_ON(start > eb->len);
4912         WARN_ON(start + len > eb->start + eb->len);
4913
4914         offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4915
4916         while (len > 0) {
4917                 page = extent_buffer_page(eb, i);
4918
4919                 cur = min(len, (PAGE_CACHE_SIZE - offset));
4920                 kaddr = page_address(page);
4921                 memcpy(dst, kaddr + offset, cur);
4922
4923                 dst += cur;
4924                 len -= cur;
4925                 offset = 0;
4926                 i++;
4927         }
4928 }
4929
4930 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
4931                                unsigned long min_len, char **map,
4932                                unsigned long *map_start,
4933                                unsigned long *map_len)
4934 {
4935         size_t offset = start & (PAGE_CACHE_SIZE - 1);
4936         char *kaddr;
4937         struct page *p;
4938         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4939         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4940         unsigned long end_i = (start_offset + start + min_len - 1) >>
4941                 PAGE_CACHE_SHIFT;
4942
4943         if (i != end_i)
4944                 return -EINVAL;
4945
4946         if (i == 0) {
4947                 offset = start_offset;
4948                 *map_start = 0;
4949         } else {
4950                 offset = 0;
4951                 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4952         }
4953
4954         if (start + min_len > eb->len) {
4955                 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4956                        "wanted %lu %lu\n",
4957                        eb->start, eb->len, start, min_len);
4958                 return -EINVAL;
4959         }
4960
4961         p = extent_buffer_page(eb, i);
4962         kaddr = page_address(p);
4963         *map = kaddr + offset;
4964         *map_len = PAGE_CACHE_SIZE - offset;
4965         return 0;
4966 }
4967
4968 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4969                           unsigned long start,
4970                           unsigned long len)
4971 {
4972         size_t cur;
4973         size_t offset;
4974         struct page *page;
4975         char *kaddr;
4976         char *ptr = (char *)ptrv;
4977         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4978         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4979         int ret = 0;
4980
4981         WARN_ON(start > eb->len);
4982         WARN_ON(start + len > eb->start + eb->len);
4983
4984         offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4985
4986         while (len > 0) {
4987                 page = extent_buffer_page(eb, i);
4988
4989                 cur = min(len, (PAGE_CACHE_SIZE - offset));
4990
4991                 kaddr = page_address(page);
4992                 ret = memcmp(ptr, kaddr + offset, cur);
4993                 if (ret)
4994                         break;
4995
4996                 ptr += cur;
4997                 len -= cur;
4998                 offset = 0;
4999                 i++;
5000         }
5001         return ret;
5002 }
5003
5004 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5005                          unsigned long start, unsigned long len)
5006 {
5007         size_t cur;
5008         size_t offset;
5009         struct page *page;
5010         char *kaddr;
5011         char *src = (char *)srcv;
5012         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5013         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5014
5015         WARN_ON(start > eb->len);
5016         WARN_ON(start + len > eb->start + eb->len);
5017
5018         offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5019
5020         while (len > 0) {
5021                 page = extent_buffer_page(eb, i);
5022                 WARN_ON(!PageUptodate(page));
5023
5024                 cur = min(len, PAGE_CACHE_SIZE - offset);
5025                 kaddr = page_address(page);
5026                 memcpy(kaddr + offset, src, cur);
5027
5028                 src += cur;
5029                 len -= cur;
5030                 offset = 0;
5031                 i++;
5032         }
5033 }
5034
5035 void memset_extent_buffer(struct extent_buffer *eb, char c,
5036                           unsigned long start, unsigned long len)
5037 {
5038         size_t cur;
5039         size_t offset;
5040         struct page *page;
5041         char *kaddr;
5042         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5043         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5044
5045         WARN_ON(start > eb->len);
5046         WARN_ON(start + len > eb->start + eb->len);
5047
5048         offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5049
5050         while (len > 0) {
5051                 page = extent_buffer_page(eb, i);
5052                 WARN_ON(!PageUptodate(page));
5053
5054                 cur = min(len, PAGE_CACHE_SIZE - offset);
5055                 kaddr = page_address(page);
5056                 memset(kaddr + offset, c, cur);
5057
5058                 len -= cur;
5059                 offset = 0;
5060                 i++;
5061         }
5062 }
5063
5064 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5065                         unsigned long dst_offset, unsigned long src_offset,
5066                         unsigned long len)
5067 {
5068         u64 dst_len = dst->len;
5069         size_t cur;
5070         size_t offset;
5071         struct page *page;
5072         char *kaddr;
5073         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5074         unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5075
5076         WARN_ON(src->len != dst_len);
5077
5078         offset = (start_offset + dst_offset) &
5079                 (PAGE_CACHE_SIZE - 1);
5080
5081         while (len > 0) {
5082                 page = extent_buffer_page(dst, i);
5083                 WARN_ON(!PageUptodate(page));
5084
5085                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5086
5087                 kaddr = page_address(page);
5088                 read_extent_buffer(src, kaddr + offset, src_offset, cur);
5089
5090                 src_offset += cur;
5091                 len -= cur;
5092                 offset = 0;
5093                 i++;
5094         }
5095 }
5096
5097 static void move_pages(struct page *dst_page, struct page *src_page,
5098                        unsigned long dst_off, unsigned long src_off,
5099                        unsigned long len)
5100 {
5101         char *dst_kaddr = page_address(dst_page);
5102         if (dst_page == src_page) {
5103                 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
5104         } else {
5105                 char *src_kaddr = page_address(src_page);
5106                 char *p = dst_kaddr + dst_off + len;
5107                 char *s = src_kaddr + src_off + len;
5108
5109                 while (len--)
5110                         *--p = *--s;
5111         }
5112 }
5113
5114 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5115 {
5116         unsigned long distance = (src > dst) ? src - dst : dst - src;
5117         return distance < len;
5118 }
5119
5120 static void copy_pages(struct page *dst_page, struct page *src_page,
5121                        unsigned long dst_off, unsigned long src_off,
5122                        unsigned long len)
5123 {
5124         char *dst_kaddr = page_address(dst_page);
5125         char *src_kaddr;
5126         int must_memmove = 0;
5127
5128         if (dst_page != src_page) {
5129                 src_kaddr = page_address(src_page);
5130         } else {
5131                 src_kaddr = dst_kaddr;
5132                 if (areas_overlap(src_off, dst_off, len))
5133                         must_memmove = 1;
5134         }
5135
5136         if (must_memmove)
5137                 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5138         else
5139                 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
5140 }
5141
5142 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5143                            unsigned long src_offset, unsigned long len)
5144 {
5145         size_t cur;
5146         size_t dst_off_in_page;
5147         size_t src_off_in_page;
5148         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5149         unsigned long dst_i;
5150         unsigned long src_i;
5151
5152         if (src_offset + len > dst->len) {
5153                 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5154                        "len %lu dst len %lu\n", src_offset, len, dst->len);
5155                 BUG_ON(1);
5156         }
5157         if (dst_offset + len > dst->len) {
5158                 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5159                        "len %lu dst len %lu\n", dst_offset, len, dst->len);
5160                 BUG_ON(1);
5161         }
5162
5163         while (len > 0) {
5164                 dst_off_in_page = (start_offset + dst_offset) &
5165                         (PAGE_CACHE_SIZE - 1);
5166                 src_off_in_page = (start_offset + src_offset) &
5167                         (PAGE_CACHE_SIZE - 1);
5168
5169                 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5170                 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5171
5172                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5173                                                src_off_in_page));
5174                 cur = min_t(unsigned long, cur,
5175                         (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5176
5177                 copy_pages(extent_buffer_page(dst, dst_i),
5178                            extent_buffer_page(dst, src_i),
5179                            dst_off_in_page, src_off_in_page, cur);
5180
5181                 src_offset += cur;
5182                 dst_offset += cur;
5183                 len -= cur;
5184         }
5185 }
5186
5187 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5188                            unsigned long src_offset, unsigned long len)
5189 {
5190         size_t cur;
5191         size_t dst_off_in_page;
5192         size_t src_off_in_page;
5193         unsigned long dst_end = dst_offset + len - 1;
5194         unsigned long src_end = src_offset + len - 1;
5195         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5196         unsigned long dst_i;
5197         unsigned long src_i;
5198
5199         if (src_offset + len > dst->len) {
5200                 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5201                        "len %lu len %lu\n", src_offset, len, dst->len);
5202                 BUG_ON(1);
5203         }
5204         if (dst_offset + len > dst->len) {
5205                 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5206                        "len %lu len %lu\n", dst_offset, len, dst->len);
5207                 BUG_ON(1);
5208         }
5209         if (dst_offset < src_offset) {
5210                 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5211                 return;
5212         }
5213         while (len > 0) {
5214                 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5215                 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5216
5217                 dst_off_in_page = (start_offset + dst_end) &
5218                         (PAGE_CACHE_SIZE - 1);
5219                 src_off_in_page = (start_offset + src_end) &
5220                         (PAGE_CACHE_SIZE - 1);
5221
5222                 cur = min_t(unsigned long, len, src_off_in_page + 1);
5223                 cur = min(cur, dst_off_in_page + 1);
5224                 move_pages(extent_buffer_page(dst, dst_i),
5225                            extent_buffer_page(dst, src_i),
5226                            dst_off_in_page - cur + 1,
5227                            src_off_in_page - cur + 1, cur);
5228
5229                 dst_end -= cur;
5230                 src_end -= cur;
5231                 len -= cur;
5232         }
5233 }
5234
5235 int try_release_extent_buffer(struct page *page)
5236 {
5237         struct extent_buffer *eb;
5238
5239         /*
5240          * We need to make sure noboody is attaching this page to an eb right
5241          * now.
5242          */
5243         spin_lock(&page->mapping->private_lock);
5244         if (!PagePrivate(page)) {
5245                 spin_unlock(&page->mapping->private_lock);
5246                 return 1;
5247         }
5248
5249         eb = (struct extent_buffer *)page->private;
5250         BUG_ON(!eb);
5251
5252         /*
5253          * This is a little awful but should be ok, we need to make sure that
5254          * the eb doesn't disappear out from under us while we're looking at
5255          * this page.
5256          */
5257         spin_lock(&eb->refs_lock);
5258         if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5259                 spin_unlock(&eb->refs_lock);
5260                 spin_unlock(&page->mapping->private_lock);
5261                 return 0;
5262         }
5263         spin_unlock(&page->mapping->private_lock);
5264
5265         /*
5266          * If tree ref isn't set then we know the ref on this eb is a real ref,
5267          * so just return, this page will likely be freed soon anyway.
5268          */
5269         if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5270                 spin_unlock(&eb->refs_lock);
5271                 return 0;
5272         }
5273
5274         return release_extent_buffer(eb);
5275 }