fs/nilfs2/btree.c

   1 /*
   2  * btree.c - NILFS B-tree.
   3  *
   4  * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  19  *
  20  * Written by Koji Sato <koji@osrg.net>.
  21  */
  22
  23 #include <linux/slab.h>
  24 #include <linux/string.h>
  25 #include <linux/errno.h>
  26 #include <linux/pagevec.h>
  27 #include "nilfs.h"
  28 #include "page.h"
  29 #include "btnode.h"
  30 #include "btree.h"
  31 #include "alloc.h"
  32 #include "dat.h"
  33
  34 static struct nilfs_btree_path *nilfs_btree_alloc_path(void)
  35 {
  36         struct nilfs_btree_path *path;
  37         int level = NILFS_BTREE_LEVEL_DATA;
  38
  39         path = kmem_cache_alloc(nilfs_btree_path_cache, GFP_NOFS);
  40         if (path == NULL)
  41                 goto out;
  42
  43         for (; level < NILFS_BTREE_LEVEL_MAX; level++) {
  44                 path[level].bp_bh = NULL;
  45                 path[level].bp_sib_bh = NULL;
  46                 path[level].bp_index = 0;
  47                 path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
  48                 path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
  49                 path[level].bp_op = NULL;
  50         }
  51
  52 out:
  53         return path;
  54 }
  55
  56 static void nilfs_btree_free_path(struct nilfs_btree_path *path)
  57 {
  58         int level = NILFS_BTREE_LEVEL_DATA;
  59
  60         for (; level < NILFS_BTREE_LEVEL_MAX; level++)
  61                 brelse(path[level].bp_bh);
  62
  63         kmem_cache_free(nilfs_btree_path_cache, path);
  64 }
  65
  66 /*
  67  * B-tree node operations
  68  */
  69 static int nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr,
  70                                  struct buffer_head **bhp)
  71 {
  72         struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache;
  73         struct buffer_head *bh;
  74         int err;
  75
  76         err = nilfs_btnode_submit_block(btnc, ptr, 0, bhp);
  77         if (err)
  78                 return err == -EEXIST ? 0 : err;
  79
  80         bh = *bhp;
  81         wait_on_buffer(bh);
  82         if (!buffer_uptodate(bh)) {
  83                 brelse(bh);
  84                 return -EIO;
  85         }
  86         if (nilfs_btree_broken_node_block(bh)) {
  87                 clear_buffer_uptodate(bh);
  88                 brelse(bh);
  89                 return -EINVAL;
  90         }
  91         return 0;
  92 }
  93
  94 static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree,
  95                                      __u64 ptr, struct buffer_head **bhp)
  96 {
  97         struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache;
  98         struct buffer_head *bh;
  99
 100         bh = nilfs_btnode_create_block(btnc, ptr);
 101         if (!bh)
 102                 return -ENOMEM;
 103
 104         set_buffer_nilfs_volatile(bh);
 105         *bhp = bh;
 106         return 0;
 107 }
 108
 109 static inline int
 110 nilfs_btree_node_get_flags(const struct nilfs_btree_node *node)
 111 {
 112         return node->bn_flags;
 113 }
 114
 115 static inline void
 116 nilfs_btree_node_set_flags(struct nilfs_btree_node *node, int flags)
 117 {
 118         node->bn_flags = flags;
 119 }
 120
 121 static inline int nilfs_btree_node_root(const struct nilfs_btree_node *node)
 122 {
 123         return nilfs_btree_node_get_flags(node) & NILFS_BTREE_NODE_ROOT;
 124 }
 125
 126 static inline int
 127 nilfs_btree_node_get_level(const struct nilfs_btree_node *node)
 128 {
 129         return node->bn_level;
 130 }
 131
 132 static inline void
 133 nilfs_btree_node_set_level(struct nilfs_btree_node *node, int level)
 134 {
 135         node->bn_level = level;
 136 }
 137
 138 static inline int
 139 nilfs_btree_node_get_nchildren(const struct nilfs_btree_node *node)
 140 {
 141         return le16_to_cpu(node->bn_nchildren);
 142 }
 143
 144 static inline void
 145 nilfs_btree_node_set_nchildren(struct nilfs_btree_node *node, int nchildren)
 146 {
 147         node->bn_nchildren = cpu_to_le16(nchildren);
 148 }
 149
 150 static inline int nilfs_btree_node_size(const struct nilfs_bmap *btree)
 151 {
 152         return 1 << btree->b_inode->i_blkbits;
 153 }
 154
 155 static int nilfs_btree_nchildren_per_block(const struct nilfs_bmap *btree)
 156 {
 157         return btree->b_nchildren_per_block;
 158 }
 159
 160 static inline __le64 *
 161 nilfs_btree_node_dkeys(const struct nilfs_btree_node *node)
 162 {
 163         return (__le64 *)((char *)(node + 1) +
 164                           (nilfs_btree_node_root(node) ?
 165                            0 : NILFS_BTREE_NODE_EXTRA_PAD_SIZE));
 166 }
 167
 168 static inline __le64 *
 169 nilfs_btree_node_dptrs(const struct nilfs_btree_node *node, int ncmax)
 170 {
 171         return (__le64 *)(nilfs_btree_node_dkeys(node) + ncmax);
 172 }
 173
 174 static inline __u64
 175 nilfs_btree_node_get_key(const struct nilfs_btree_node *node, int index)
 176 {
 177         return le64_to_cpu(*(nilfs_btree_node_dkeys(node) + index));
 178 }
 179
 180 static inline void
 181 nilfs_btree_node_set_key(struct nilfs_btree_node *node, int index, __u64 key)
 182 {
 183         *(nilfs_btree_node_dkeys(node) + index) = cpu_to_le64(key);
 184 }
 185
 186 static inline __u64
 187 nilfs_btree_node_get_ptr(const struct nilfs_btree_node *node, int index,
 188                          int ncmax)
 189 {
 190         return le64_to_cpu(*(nilfs_btree_node_dptrs(node, ncmax) + index));
 191 }
 192
 193 static inline void
 194 nilfs_btree_node_set_ptr(struct nilfs_btree_node *node, int index, __u64 ptr,
 195                          int ncmax)
 196 {
 197         *(nilfs_btree_node_dptrs(node, ncmax) + index) = cpu_to_le64(ptr);
 198 }
 199
 200 static void nilfs_btree_node_init(struct nilfs_btree_node *node, int flags,
 201                                   int level, int nchildren, int ncmax,
 202                                   const __u64 *keys, const __u64 *ptrs)
 203 {
 204         __le64 *dkeys;
 205         __le64 *dptrs;
 206         int i;
 207
 208         nilfs_btree_node_set_flags(node, flags);
 209         nilfs_btree_node_set_level(node, level);
 210         nilfs_btree_node_set_nchildren(node, nchildren);
 211
 212         dkeys = nilfs_btree_node_dkeys(node);
 213         dptrs = nilfs_btree_node_dptrs(node, ncmax);
 214         for (i = 0; i < nchildren; i++) {
 215                 dkeys[i] = cpu_to_le64(keys[i]);
 216                 dptrs[i] = cpu_to_le64(ptrs[i]);
 217         }
 218 }
 219
 220 /* Assume the buffer heads corresponding to left and right are locked. */
 221 static void nilfs_btree_node_move_left(struct nilfs_btree_node *left,
 222                                        struct nilfs_btree_node *right,
 223                                        int n, int lncmax, int rncmax)
 224 {
 225         __le64 *ldkeys, *rdkeys;
 226         __le64 *ldptrs, *rdptrs;
 227         int lnchildren, rnchildren;
 228
 229         ldkeys = nilfs_btree_node_dkeys(left);
 230         ldptrs = nilfs_btree_node_dptrs(left, lncmax);
 231         lnchildren = nilfs_btree_node_get_nchildren(left);
 232
 233         rdkeys = nilfs_btree_node_dkeys(right);
 234         rdptrs = nilfs_btree_node_dptrs(right, rncmax);
 235         rnchildren = nilfs_btree_node_get_nchildren(right);
 236
 237         memcpy(ldkeys + lnchildren, rdkeys, n * sizeof(*rdkeys));
 238         memcpy(ldptrs + lnchildren, rdptrs, n * sizeof(*rdptrs));
 239         memmove(rdkeys, rdkeys + n, (rnchildren - n) * sizeof(*rdkeys));
 240         memmove(rdptrs, rdptrs + n, (rnchildren - n) * sizeof(*rdptrs));
 241
 242         lnchildren += n;
 243         rnchildren -= n;
 244         nilfs_btree_node_set_nchildren(left, lnchildren);
 245         nilfs_btree_node_set_nchildren(right, rnchildren);
 246 }
 247
 248 /* Assume that the buffer heads corresponding to left and right are locked. */
 249 static void nilfs_btree_node_move_right(struct nilfs_btree_node *left,
 250                                         struct nilfs_btree_node *right,
 251                                         int n, int lncmax, int rncmax)
 252 {
 253         __le64 *ldkeys, *rdkeys;
 254         __le64 *ldptrs, *rdptrs;
 255         int lnchildren, rnchildren;
 256
 257         ldkeys = nilfs_btree_node_dkeys(left);
 258         ldptrs = nilfs_btree_node_dptrs(left, lncmax);
 259         lnchildren = nilfs_btree_node_get_nchildren(left);
 260
 261         rdkeys = nilfs_btree_node_dkeys(right);
 262         rdptrs = nilfs_btree_node_dptrs(right, rncmax);
 263         rnchildren = nilfs_btree_node_get_nchildren(right);
 264
 265         memmove(rdkeys + n, rdkeys, rnchildren * sizeof(*rdkeys));
 266         memmove(rdptrs + n, rdptrs, rnchildren * sizeof(*rdptrs));
 267         memcpy(rdkeys, ldkeys + lnchildren - n, n * sizeof(*rdkeys));
 268         memcpy(rdptrs, ldptrs + lnchildren - n, n * sizeof(*rdptrs));
 269
 270         lnchildren -= n;
 271         rnchildren += n;
 272         nilfs_btree_node_set_nchildren(left, lnchildren);
 273         nilfs_btree_node_set_nchildren(right, rnchildren);
 274 }
 275
 276 /* Assume that the buffer head corresponding to node is locked. */
 277 static void nilfs_btree_node_insert(struct nilfs_btree_node *node, int index,
 278                                     __u64 key, __u64 ptr, int ncmax)
 279 {
 280         __le64 *dkeys;
 281         __le64 *dptrs;
 282         int nchildren;
 283
 284         dkeys = nilfs_btree_node_dkeys(node);
 285         dptrs = nilfs_btree_node_dptrs(node, ncmax);
 286         nchildren = nilfs_btree_node_get_nchildren(node);
 287         if (index < nchildren) {
 288                 memmove(dkeys + index + 1, dkeys + index,
 289                         (nchildren - index) * sizeof(*dkeys));
 290                 memmove(dptrs + index + 1, dptrs + index,
 291                         (nchildren - index) * sizeof(*dptrs));
 292         }
 293         dkeys[index] = cpu_to_le64(key);
 294         dptrs[index] = cpu_to_le64(ptr);
 295         nchildren++;
 296         nilfs_btree_node_set_nchildren(node, nchildren);
 297 }
 298
 299 /* Assume that the buffer head corresponding to node is locked. */
 300 static void nilfs_btree_node_delete(struct nilfs_btree_node *node, int index,
 301                                     __u64 *keyp, __u64 *ptrp, int ncmax)
 302 {
 303         __u64 key;
 304         __u64 ptr;
 305         __le64 *dkeys;
 306         __le64 *dptrs;
 307         int nchildren;
 308
 309         dkeys = nilfs_btree_node_dkeys(node);
 310         dptrs = nilfs_btree_node_dptrs(node, ncmax);
 311         key = le64_to_cpu(dkeys[index]);
 312         ptr = le64_to_cpu(dptrs[index]);
 313         nchildren = nilfs_btree_node_get_nchildren(node);
 314         if (keyp != NULL)
 315                 *keyp = key;
 316         if (ptrp != NULL)
 317                 *ptrp = ptr;
 318
 319         if (index < nchildren - 1) {
 320                 memmove(dkeys + index, dkeys + index + 1,
 321                         (nchildren - index - 1) * sizeof(*dkeys));
 322                 memmove(dptrs + index, dptrs + index + 1,
 323                         (nchildren - index - 1) * sizeof(*dptrs));
 324         }
 325         nchildren--;
 326         nilfs_btree_node_set_nchildren(node, nchildren);
 327 }
 328
 329 static int nilfs_btree_node_lookup(const struct nilfs_btree_node *node,
 330                                    __u64 key, int *indexp)
 331 {
 332         __u64 nkey;
 333         int index, low, high, s;
 334
 335         /* binary search */
 336         low = 0;
 337         high = nilfs_btree_node_get_nchildren(node) - 1;
 338         index = 0;
 339         s = 0;
 340         while (low <= high) {
 341                 index = (low + high) / 2;
 342                 nkey = nilfs_btree_node_get_key(node, index);
 343                 if (nkey == key) {
 344                         s = 0;
 345                         goto out;
 346                 } else if (nkey < key) {
 347                         low = index + 1;
 348                         s = -1;
 349                 } else {
 350                         high = index - 1;
 351                         s = 1;
 352                 }
 353         }
 354
 355         /* adjust index */
 356         if (nilfs_btree_node_get_level(node) > NILFS_BTREE_LEVEL_NODE_MIN) {
 357                 if (s > 0 && index > 0)
 358                         index--;
 359         } else if (s < 0)
 360                 index++;
 361
 362  out:
 363         *indexp = index;
 364
 365         return s == 0;
 366 }
 367
 368 /**
 369  * nilfs_btree_node_broken - verify consistency of btree node
 370  * @node: btree node block to be examined
 371  * @size: node size (in bytes)
 372  * @blocknr: block number
 373  *
 374  * Return Value: If node is broken, 1 is returned. Otherwise, 0 is returned.
 375  */
 376 static int nilfs_btree_node_broken(const struct nilfs_btree_node *node,
 377                                    size_t size, sector_t blocknr)
 378 {
 379         int level, flags, nchildren;
 380         int ret = 0;
 381
 382         level = nilfs_btree_node_get_level(node);
 383         flags = nilfs_btree_node_get_flags(node);
 384         nchildren = nilfs_btree_node_get_nchildren(node);
 385
 386         if (unlikely(level < NILFS_BTREE_LEVEL_NODE_MIN ||
 387                      level >= NILFS_BTREE_LEVEL_MAX ||
 388                      (flags & NILFS_BTREE_NODE_ROOT) ||
 389                      nchildren < 0 ||
 390                      nchildren > NILFS_BTREE_NODE_NCHILDREN_MAX(size))) {
 391                 printk(KERN_CRIT "NILFS: bad btree node (blocknr=%llu): "
 392                        "level = %d, flags = 0x%x, nchildren = %d\n",
 393                        (unsigned long long)blocknr, level, flags, nchildren);
 394                 ret = 1;
 395         }
 396         return ret;
 397 }
 398
 399 int nilfs_btree_broken_node_block(struct buffer_head *bh)
 400 {
 401         return nilfs_btree_node_broken((struct nilfs_btree_node *)bh->b_data,
 402                                        bh->b_size, bh->b_blocknr);
 403 }
 404
 405 static inline struct nilfs_btree_node *
 406 nilfs_btree_get_root(const struct nilfs_bmap *btree)
 407 {
 408         return (struct nilfs_btree_node *)btree->b_u.u_data;
 409 }
 410
 411 static inline struct nilfs_btree_node *
 412 nilfs_btree_get_nonroot_node(const struct nilfs_btree_path *path, int level)
 413 {
 414         return (struct nilfs_btree_node *)path[level].bp_bh->b_data;
 415 }
 416
 417 static inline struct nilfs_btree_node *
 418 nilfs_btree_get_sib_node(const struct nilfs_btree_path *path, int level)
 419 {
 420         return (struct nilfs_btree_node *)path[level].bp_sib_bh->b_data;
 421 }
 422
 423 static inline int nilfs_btree_height(const struct nilfs_bmap *btree)
 424 {
 425         return nilfs_btree_node_get_level(nilfs_btree_get_root(btree)) + 1;
 426 }
 427
 428 static struct nilfs_btree_node *
 429 nilfs_btree_get_node(const struct nilfs_bmap *btree,
 430                      const struct nilfs_btree_path *path,
 431                      int level, int *ncmaxp)
 432 {
 433         struct nilfs_btree_node *node;
 434
 435         if (level == nilfs_btree_height(btree) - 1) {
 436                 node = nilfs_btree_get_root(btree);
 437                 *ncmaxp = NILFS_BTREE_ROOT_NCHILDREN_MAX;
 438         } else {
 439                 node = nilfs_btree_get_nonroot_node(path, level);
 440                 *ncmaxp = nilfs_btree_nchildren_per_block(btree);
 441         }
 442         return node;
 443 }
 444
 445 static inline int
 446 nilfs_btree_bad_node(struct nilfs_btree_node *node, int level)
 447 {
 448         if (unlikely(nilfs_btree_node_get_level(node) != level)) {
 449                 dump_stack();
 450                 printk(KERN_CRIT "NILFS: btree level mismatch: %d != %d\n",
 451                        nilfs_btree_node_get_level(node), level);
 452                 return 1;
 453         }
 454         return 0;
 455 }
 456
 457 static int nilfs_btree_do_lookup(const struct nilfs_bmap *btree,
 458                                  struct nilfs_btree_path *path,
 459                                  __u64 key, __u64 *ptrp, int minlevel)
 460 {
 461         struct nilfs_btree_node *node;
 462         __u64 ptr;
 463         int level, index, found, ncmax, ret;
 464
 465         node = nilfs_btree_get_root(btree);
 466         level = nilfs_btree_node_get_level(node);
 467         if (level < minlevel || nilfs_btree_node_get_nchildren(node) <= 0)
 468                 return -ENOENT;
 469
 470         found = nilfs_btree_node_lookup(node, key, &index);
 471         ptr = nilfs_btree_node_get_ptr(node, index,
 472                                        NILFS_BTREE_ROOT_NCHILDREN_MAX);
 473         path[level].bp_bh = NULL;
 474         path[level].bp_index = index;
 475
 476         ncmax = nilfs_btree_nchildren_per_block(btree);
 477
 478         for (level--; level >= minlevel; level--) {
 479                 ret = nilfs_btree_get_block(btree, ptr, &path[level].bp_bh);
 480                 if (ret < 0)
 481                         return ret;
 482                 node = nilfs_btree_get_nonroot_node(path, level);
 483                 if (nilfs_btree_bad_node(node, level))
 484                         return -EINVAL;
 485                 if (!found)
 486                         found = nilfs_btree_node_lookup(node, key, &index);
 487                 else
 488                         index = 0;
 489                 if (index < ncmax) {
 490                         ptr = nilfs_btree_node_get_ptr(node, index, ncmax);
 491                 } else {
 492                         WARN_ON(found || level != NILFS_BTREE_LEVEL_NODE_MIN);
 493                         /* insert */
 494                         ptr = NILFS_BMAP_INVALID_PTR;
 495                 }
 496                 path[level].bp_index = index;
 497         }
 498         if (!found)
 499                 return -ENOENT;
 500
 501         if (ptrp != NULL)
 502                 *ptrp = ptr;
 503
 504         return 0;
 505 }
 506
 507 static int nilfs_btree_do_lookup_last(const struct nilfs_bmap *btree,
 508                                       struct nilfs_btree_path *path,
 509                                       __u64 *keyp, __u64 *ptrp)
 510 {
 511         struct nilfs_btree_node *node;
 512         __u64 ptr;
 513         int index, level, ncmax, ret;
 514
 515         node = nilfs_btree_get_root(btree);
 516         index = nilfs_btree_node_get_nchildren(node) - 1;
 517         if (index < 0)
 518                 return -ENOENT;
 519         level = nilfs_btree_node_get_level(node);
 520         ptr = nilfs_btree_node_get_ptr(node, index,
 521                                        NILFS_BTREE_ROOT_NCHILDREN_MAX);
 522         path[level].bp_bh = NULL;
 523         path[level].bp_index = index;
 524         ncmax = nilfs_btree_nchildren_per_block(btree);
 525
 526         for (level--; level > 0; level--) {
 527                 ret = nilfs_btree_get_block(btree, ptr, &path[level].bp_bh);
 528                 if (ret < 0)
 529                         return ret;
 530                 node = nilfs_btree_get_nonroot_node(path, level);
 531                 if (nilfs_btree_bad_node(node, level))
 532                         return -EINVAL;
 533                 index = nilfs_btree_node_get_nchildren(node) - 1;
 534                 ptr = nilfs_btree_node_get_ptr(node, index, ncmax);
 535                 path[level].bp_index = index;
 536         }
 537
 538         if (keyp != NULL)
 539                 *keyp = nilfs_btree_node_get_key(node, index);
 540         if (ptrp != NULL)
 541                 *ptrp = ptr;
 542
 543         return 0;
 544 }
 545
 546 static int nilfs_btree_lookup(const struct nilfs_bmap *btree,
 547                               __u64 key, int level, __u64 *ptrp)
 548 {
 549         struct nilfs_btree_path *path;
 550         int ret;
 551
 552         path = nilfs_btree_alloc_path();
 553         if (path == NULL)
 554                 return -ENOMEM;
 555
 556         ret = nilfs_btree_do_lookup(btree, path, key, ptrp, level);
 557
 558         nilfs_btree_free_path(path);
 559
 560         return ret;
 561 }
 562
 563 static int nilfs_btree_lookup_contig(const struct nilfs_bmap *btree,
 564                                      __u64 key, __u64 *ptrp, unsigned maxblocks)
 565 {
 566         struct nilfs_btree_path *path;
 567         struct nilfs_btree_node *node;
 568         struct inode *dat = NULL;
 569         __u64 ptr, ptr2;
 570         sector_t blocknr;
 571         int level = NILFS_BTREE_LEVEL_NODE_MIN;
 572         int ret, cnt, index, maxlevel, ncmax;
 573
 574         path = nilfs_btree_alloc_path();
 575         if (path == NULL)
 576                 return -ENOMEM;
 577
 578         ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level);
 579         if (ret < 0)
 580                 goto out;
 581
 582         if (NILFS_BMAP_USE_VBN(btree)) {
 583                 dat = nilfs_bmap_get_dat(btree);
 584                 ret = nilfs_dat_translate(dat, ptr, &blocknr);
 585                 if (ret < 0)
 586                         goto out;
 587                 ptr = blocknr;
 588         }
 589         cnt = 1;
 590         if (cnt == maxblocks)
 591                 goto end;
 592
 593         maxlevel = nilfs_btree_height(btree) - 1;
 594         node = nilfs_btree_get_node(btree, path, level, &ncmax);
 595         index = path[level].bp_index + 1;
 596         for (;;) {
 597                 while (index < nilfs_btree_node_get_nchildren(node)) {
 598                         if (nilfs_btree_node_get_key(node, index) !=
 599                             key + cnt)
 600                                 goto end;
 601                         ptr2 = nilfs_btree_node_get_ptr(node, index, ncmax);
 602                         if (dat) {
 603                                 ret = nilfs_dat_translate(dat, ptr2, &blocknr);
 604                                 if (ret < 0)
 605                                         goto out;
 606                                 ptr2 = blocknr;
 607                         }
 608                         if (ptr2 != ptr + cnt || ++cnt == maxblocks)
 609                                 goto end;
 610                         index++;
 611                         continue;
 612                 }
 613                 if (level == maxlevel)
 614                         break;
 615
 616                 /* look-up right sibling node */
 617                 node = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
 618                 index = path[level + 1].bp_index + 1;
 619                 if (index >= nilfs_btree_node_get_nchildren(node) ||
 620                     nilfs_btree_node_get_key(node, index) != key + cnt)
 621                         break;
 622                 ptr2 = nilfs_btree_node_get_ptr(node, index, ncmax);
 623                 path[level + 1].bp_index = index;
 624
 625                 brelse(path[level].bp_bh);
 626                 path[level].bp_bh = NULL;
 627                 ret = nilfs_btree_get_block(btree, ptr2, &path[level].bp_bh);
 628                 if (ret < 0)
 629                         goto out;
 630                 node = nilfs_btree_get_nonroot_node(path, level);
 631                 ncmax = nilfs_btree_nchildren_per_block(btree);
 632                 index = 0;
 633                 path[level].bp_index = index;
 634         }
 635  end:
 636         *ptrp = ptr;
 637         ret = cnt;
 638  out:
 639         nilfs_btree_free_path(path);
 640         return ret;
 641 }
 642
 643 static void nilfs_btree_promote_key(struct nilfs_bmap *btree,
 644                                     struct nilfs_btree_path *path,
 645                                     int level, __u64 key)
 646 {
 647         if (level < nilfs_btree_height(btree) - 1) {
 648                 do {
 649                         nilfs_btree_node_set_key(
 650                                 nilfs_btree_get_nonroot_node(path, level),
 651                                 path[level].bp_index, key);
 652                         if (!buffer_dirty(path[level].bp_bh))
 653                                 nilfs_btnode_mark_dirty(path[level].bp_bh);
 654                 } while ((path[level].bp_index == 0) &&
 655                          (++level < nilfs_btree_height(btree) - 1));
 656         }
 657
 658         /* root */
 659         if (level == nilfs_btree_height(btree) - 1) {
 660                 nilfs_btree_node_set_key(nilfs_btree_get_root(btree),
 661                                          path[level].bp_index, key);
 662         }
 663 }
 664
 665 static void nilfs_btree_do_insert(struct nilfs_bmap *btree,
 666                                   struct nilfs_btree_path *path,
 667                                   int level, __u64 *keyp, __u64 *ptrp)
 668 {
 669         struct nilfs_btree_node *node;
 670         int ncblk;
 671
 672         if (level < nilfs_btree_height(btree) - 1) {
 673                 node = nilfs_btree_get_nonroot_node(path, level);
 674                 ncblk = nilfs_btree_nchildren_per_block(btree);
 675                 nilfs_btree_node_insert(node, path[level].bp_index,
 676                                         *keyp, *ptrp, ncblk);
 677                 if (!buffer_dirty(path[level].bp_bh))
 678                         nilfs_btnode_mark_dirty(path[level].bp_bh);
 679
 680                 if (path[level].bp_index == 0)
 681                         nilfs_btree_promote_key(btree, path, level + 1,
 682                                                 nilfs_btree_node_get_key(node,
 683                                                                          0));
 684         } else {
 685                 node = nilfs_btree_get_root(btree);
 686                 nilfs_btree_node_insert(node, path[level].bp_index,
 687                                         *keyp, *ptrp,
 688                                         NILFS_BTREE_ROOT_NCHILDREN_MAX);
 689         }
 690 }
 691
 692 static void nilfs_btree_carry_left(struct nilfs_bmap *btree,
 693                                    struct nilfs_btree_path *path,
 694                                    int level, __u64 *keyp, __u64 *ptrp)
 695 {
 696         struct nilfs_btree_node *node, *left;
 697         int nchildren, lnchildren, n, move, ncblk;
 698
 699         node = nilfs_btree_get_nonroot_node(path, level);
 700         left = nilfs_btree_get_sib_node(path, level);
 701         nchildren = nilfs_btree_node_get_nchildren(node);
 702         lnchildren = nilfs_btree_node_get_nchildren(left);
 703         ncblk = nilfs_btree_nchildren_per_block(btree);
 704         move = 0;
 705
 706         n = (nchildren + lnchildren + 1) / 2 - lnchildren;
 707         if (n > path[level].bp_index) {
 708                 /* move insert point */
 709                 n--;
 710                 move = 1;
 711         }
 712
 713         nilfs_btree_node_move_left(left, node, n, ncblk, ncblk);
 714
 715         if (!buffer_dirty(path[level].bp_bh))
 716                 nilfs_btnode_mark_dirty(path[level].bp_bh);
 717         if (!buffer_dirty(path[level].bp_sib_bh))
 718                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
 719
 720         nilfs_btree_promote_key(btree, path, level + 1,
 721                                 nilfs_btree_node_get_key(node, 0));
 722
 723         if (move) {
 724                 brelse(path[level].bp_bh);
 725                 path[level].bp_bh = path[level].bp_sib_bh;
 726                 path[level].bp_sib_bh = NULL;
 727                 path[level].bp_index += lnchildren;
 728                 path[level + 1].bp_index--;
 729         } else {
 730                 brelse(path[level].bp_sib_bh);
 731                 path[level].bp_sib_bh = NULL;
 732                 path[level].bp_index -= n;
 733         }
 734
 735         nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
 736 }
 737
 738 static void nilfs_btree_carry_right(struct nilfs_bmap *btree,
 739                                     struct nilfs_btree_path *path,
 740                                     int level, __u64 *keyp, __u64 *ptrp)
 741 {
 742         struct nilfs_btree_node *node, *right;
 743         int nchildren, rnchildren, n, move, ncblk;
 744
 745         node = nilfs_btree_get_nonroot_node(path, level);
 746         right = nilfs_btree_get_sib_node(path, level);
 747         nchildren = nilfs_btree_node_get_nchildren(node);
 748         rnchildren = nilfs_btree_node_get_nchildren(right);
 749         ncblk = nilfs_btree_nchildren_per_block(btree);
 750         move = 0;
 751
 752         n = (nchildren + rnchildren + 1) / 2 - rnchildren;
 753         if (n > nchildren - path[level].bp_index) {
 754                 /* move insert point */
 755                 n--;
 756                 move = 1;
 757         }
 758
 759         nilfs_btree_node_move_right(node, right, n, ncblk, ncblk);
 760
 761         if (!buffer_dirty(path[level].bp_bh))
 762                 nilfs_btnode_mark_dirty(path[level].bp_bh);
 763         if (!buffer_dirty(path[level].bp_sib_bh))
 764                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
 765
 766         path[level + 1].bp_index++;
 767         nilfs_btree_promote_key(btree, path, level + 1,
 768                                 nilfs_btree_node_get_key(right, 0));
 769         path[level + 1].bp_index--;
 770
 771         if (move) {
 772                 brelse(path[level].bp_bh);
 773                 path[level].bp_bh = path[level].bp_sib_bh;
 774                 path[level].bp_sib_bh = NULL;
 775                 path[level].bp_index -= nilfs_btree_node_get_nchildren(node);
 776                 path[level + 1].bp_index++;
 777         } else {
 778                 brelse(path[level].bp_sib_bh);
 779                 path[level].bp_sib_bh = NULL;
 780         }
 781
 782         nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
 783 }
 784
 785 static void nilfs_btree_split(struct nilfs_bmap *btree,
 786                               struct nilfs_btree_path *path,
 787                               int level, __u64 *keyp, __u64 *ptrp)
 788 {
 789         struct nilfs_btree_node *node, *right;
 790         __u64 newkey;
 791         __u64 newptr;
 792         int nchildren, n, move, ncblk;
 793
 794         node = nilfs_btree_get_nonroot_node(path, level);
 795         right = nilfs_btree_get_sib_node(path, level);
 796         nchildren = nilfs_btree_node_get_nchildren(node);
 797         ncblk = nilfs_btree_nchildren_per_block(btree);
 798         move = 0;
 799
 800         n = (nchildren + 1) / 2;
 801         if (n > nchildren - path[level].bp_index) {
 802                 n--;
 803                 move = 1;
 804         }
 805
 806         nilfs_btree_node_move_right(node, right, n, ncblk, ncblk);
 807
 808         if (!buffer_dirty(path[level].bp_bh))
 809                 nilfs_btnode_mark_dirty(path[level].bp_bh);
 810         if (!buffer_dirty(path[level].bp_sib_bh))
 811                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
 812
 813         newkey = nilfs_btree_node_get_key(right, 0);
 814         newptr = path[level].bp_newreq.bpr_ptr;
 815
 816         if (move) {
 817                 path[level].bp_index -= nilfs_btree_node_get_nchildren(node);
 818                 nilfs_btree_node_insert(right, path[level].bp_index,
 819                                         *keyp, *ptrp, ncblk);
 820
 821                 *keyp = nilfs_btree_node_get_key(right, 0);
 822                 *ptrp = path[level].bp_newreq.bpr_ptr;
 823
 824                 brelse(path[level].bp_bh);
 825                 path[level].bp_bh = path[level].bp_sib_bh;
 826                 path[level].bp_sib_bh = NULL;
 827         } else {
 828                 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
 829
 830                 *keyp = nilfs_btree_node_get_key(right, 0);
 831                 *ptrp = path[level].bp_newreq.bpr_ptr;
 832
 833                 brelse(path[level].bp_sib_bh);
 834                 path[level].bp_sib_bh = NULL;
 835         }
 836
 837         path[level + 1].bp_index++;
 838 }
 839
 840 static void nilfs_btree_grow(struct nilfs_bmap *btree,
 841                              struct nilfs_btree_path *path,
 842                              int level, __u64 *keyp, __u64 *ptrp)
 843 {
 844         struct nilfs_btree_node *root, *child;
 845         int n, ncblk;
 846
 847         root = nilfs_btree_get_root(btree);
 848         child = nilfs_btree_get_sib_node(path, level);
 849         ncblk = nilfs_btree_nchildren_per_block(btree);
 850
 851         n = nilfs_btree_node_get_nchildren(root);
 852
 853         nilfs_btree_node_move_right(root, child, n,
 854                                     NILFS_BTREE_ROOT_NCHILDREN_MAX, ncblk);
 855         nilfs_btree_node_set_level(root, level + 1);
 856
 857         if (!buffer_dirty(path[level].bp_sib_bh))
 858                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
 859
 860         path[level].bp_bh = path[level].bp_sib_bh;
 861         path[level].bp_sib_bh = NULL;
 862
 863         nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
 864
 865         *keyp = nilfs_btree_node_get_key(child, 0);
 866         *ptrp = path[level].bp_newreq.bpr_ptr;
 867 }
 868
 869 static __u64 nilfs_btree_find_near(const struct nilfs_bmap *btree,
 870                                    const struct nilfs_btree_path *path)
 871 {
 872         struct nilfs_btree_node *node;
 873         int level, ncmax;
 874
 875         if (path == NULL)
 876                 return NILFS_BMAP_INVALID_PTR;
 877
 878         /* left sibling */
 879         level = NILFS_BTREE_LEVEL_NODE_MIN;
 880         if (path[level].bp_index > 0) {
 881                 node = nilfs_btree_get_node(btree, path, level, &ncmax);
 882                 return nilfs_btree_node_get_ptr(node,
 883                                                 path[level].bp_index - 1,
 884                                                 ncmax);
 885         }
 886
 887         /* parent */
 888         level = NILFS_BTREE_LEVEL_NODE_MIN + 1;
 889         if (level <= nilfs_btree_height(btree) - 1) {
 890                 node = nilfs_btree_get_node(btree, path, level, &ncmax);
 891                 return nilfs_btree_node_get_ptr(node, path[level].bp_index,
 892                                                 ncmax);
 893         }
 894
 895         return NILFS_BMAP_INVALID_PTR;
 896 }
 897
 898 static __u64 nilfs_btree_find_target_v(const struct nilfs_bmap *btree,
 899                                        const struct nilfs_btree_path *path,
 900                                        __u64 key)
 901 {
 902         __u64 ptr;
 903
 904         ptr = nilfs_bmap_find_target_seq(btree, key);
 905         if (ptr != NILFS_BMAP_INVALID_PTR)
 906                 /* sequential access */
 907                 return ptr;
 908         else {
 909                 ptr = nilfs_btree_find_near(btree, path);
 910                 if (ptr != NILFS_BMAP_INVALID_PTR)
 911                         /* near */
 912                         return ptr;
 913         }
 914         /* block group */
 915         return nilfs_bmap_find_target_in_group(btree);
 916 }
 917
 918 static int nilfs_btree_prepare_insert(struct nilfs_bmap *btree,
 919                                       struct nilfs_btree_path *path,
 920                                       int *levelp, __u64 key, __u64 ptr,
 921                                       struct nilfs_bmap_stats *stats)
 922 {
 923         struct buffer_head *bh;
 924         struct nilfs_btree_node *node, *parent, *sib;
 925         __u64 sibptr;
 926         int pindex, level, ncmax, ncblk, ret;
 927         struct inode *dat = NULL;
 928
 929         stats->bs_nblocks = 0;
 930         level = NILFS_BTREE_LEVEL_DATA;
 931
 932         /* allocate a new ptr for data block */
 933         if (NILFS_BMAP_USE_VBN(btree)) {
 934                 path[level].bp_newreq.bpr_ptr =
 935                         nilfs_btree_find_target_v(btree, path, key);
 936                 dat = nilfs_bmap_get_dat(btree);
 937         }
 938
 939         ret = nilfs_bmap_prepare_alloc_ptr(btree, &path[level].bp_newreq, dat);
 940         if (ret < 0)
 941                 goto err_out_data;
 942
 943         ncblk = nilfs_btree_nchildren_per_block(btree);
 944
 945         for (level = NILFS_BTREE_LEVEL_NODE_MIN;
 946              level < nilfs_btree_height(btree) - 1;
 947              level++) {
 948                 node = nilfs_btree_get_nonroot_node(path, level);
 949                 if (nilfs_btree_node_get_nchildren(node) < ncblk) {
 950                         path[level].bp_op = nilfs_btree_do_insert;
 951                         stats->bs_nblocks++;
 952                         goto out;
 953                 }
 954
 955                 parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
 956                 pindex = path[level + 1].bp_index;
 957
 958                 /* left sibling */
 959                 if (pindex > 0) {
 960                         sibptr = nilfs_btree_node_get_ptr(parent, pindex - 1,
 961                                                           ncmax);
 962                         ret = nilfs_btree_get_block(btree, sibptr, &bh);
 963                         if (ret < 0)
 964                                 goto err_out_child_node;
 965                         sib = (struct nilfs_btree_node *)bh->b_data;
 966                         if (nilfs_btree_node_get_nchildren(sib) < ncblk) {
 967                                 path[level].bp_sib_bh = bh;
 968                                 path[level].bp_op = nilfs_btree_carry_left;
 969                                 stats->bs_nblocks++;
 970                                 goto out;
 971                         } else {
 972                                 brelse(bh);
 973                         }
 974                 }
 975
 976                 /* right sibling */
 977                 if (pindex < nilfs_btree_node_get_nchildren(parent) - 1) {
 978                         sibptr = nilfs_btree_node_get_ptr(parent, pindex + 1,
 979                                                           ncmax);
 980                         ret = nilfs_btree_get_block(btree, sibptr, &bh);
 981                         if (ret < 0)
 982                                 goto err_out_child_node;
 983                         sib = (struct nilfs_btree_node *)bh->b_data;
 984                         if (nilfs_btree_node_get_nchildren(sib) < ncblk) {
 985                                 path[level].bp_sib_bh = bh;
 986                                 path[level].bp_op = nilfs_btree_carry_right;
 987                                 stats->bs_nblocks++;
 988                                 goto out;
 989                         } else {
 990                                 brelse(bh);
 991                         }
 992                 }
 993
 994                 /* split */
 995                 path[level].bp_newreq.bpr_ptr =
 996                         path[level - 1].bp_newreq.bpr_ptr + 1;
 997                 ret = nilfs_bmap_prepare_alloc_ptr(btree,
 998                                                    &path[level].bp_newreq, dat);
 999                 if (ret < 0)
1000                         goto err_out_child_node;
1001                 ret = nilfs_btree_get_new_block(btree,
1002                                                 path[level].bp_newreq.bpr_ptr,
1003                                                 &bh);
1004                 if (ret < 0)
1005                         goto err_out_curr_node;
1006
1007                 stats->bs_nblocks++;
1008
1009                 sib = (struct nilfs_btree_node *)bh->b_data;
1010                 nilfs_btree_node_init(sib, 0, level, 0, ncblk, NULL, NULL);
1011                 path[level].bp_sib_bh = bh;
1012                 path[level].bp_op = nilfs_btree_split;
1013         }
1014
1015         /* root */
1016         node = nilfs_btree_get_root(btree);
1017         if (nilfs_btree_node_get_nchildren(node) <
1018             NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1019                 path[level].bp_op = nilfs_btree_do_insert;
1020                 stats->bs_nblocks++;
1021                 goto out;
1022         }
1023
1024         /* grow */
1025         path[level].bp_newreq.bpr_ptr = path[level - 1].bp_newreq.bpr_ptr + 1;
1026         ret = nilfs_bmap_prepare_alloc_ptr(btree, &path[level].bp_newreq, dat);
1027         if (ret < 0)
1028                 goto err_out_child_node;
1029         ret = nilfs_btree_get_new_block(btree, path[level].bp_newreq.bpr_ptr,
1030                                         &bh);
1031         if (ret < 0)
1032                 goto err_out_curr_node;
1033
1034         nilfs_btree_node_init((struct nilfs_btree_node *)bh->b_data,
1035                               0, level, 0, ncblk, NULL, NULL);
1036         path[level].bp_sib_bh = bh;
1037         path[level].bp_op = nilfs_btree_grow;
1038
1039         level++;
1040         path[level].bp_op = nilfs_btree_do_insert;
1041
1042         /* a newly-created node block and a data block are added */
1043         stats->bs_nblocks += 2;
1044
1045         /* success */
1046  out:
1047         *levelp = level;
1048         return ret;
1049
1050         /* error */
1051  err_out_curr_node:
1052         nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat);
1053  err_out_child_node:
1054         for (level--; level > NILFS_BTREE_LEVEL_DATA; level--) {
1055                 nilfs_btnode_delete(path[level].bp_sib_bh);
1056                 nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat);
1057
1058         }
1059
1060         nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat);
1061  err_out_data:
1062         *levelp = level;
1063         stats->bs_nblocks = 0;
1064         return ret;
1065 }
1066
1067 static void nilfs_btree_commit_insert(struct nilfs_bmap *btree,
1068                                       struct nilfs_btree_path *path,
1069                                       int maxlevel, __u64 key, __u64 ptr)
1070 {
1071         struct inode *dat = NULL;
1072         int level;
1073
1074         set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1075         ptr = path[NILFS_BTREE_LEVEL_DATA].bp_newreq.bpr_ptr;
1076         if (NILFS_BMAP_USE_VBN(btree)) {
1077                 nilfs_bmap_set_target_v(btree, key, ptr);
1078                 dat = nilfs_bmap_get_dat(btree);
1079         }
1080
1081         for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1082                 nilfs_bmap_commit_alloc_ptr(btree,
1083                                             &path[level - 1].bp_newreq, dat);
1084                 path[level].bp_op(btree, path, level, &key, &ptr);
1085         }
1086
1087         if (!nilfs_bmap_dirty(btree))
1088                 nilfs_bmap_set_dirty(btree);
1089 }
1090
1091 static int nilfs_btree_insert(struct nilfs_bmap *btree, __u64 key, __u64 ptr)
1092 {
1093         struct nilfs_btree_path *path;
1094         struct nilfs_bmap_stats stats;
1095         int level, ret;
1096
1097         path = nilfs_btree_alloc_path();
1098         if (path == NULL)
1099                 return -ENOMEM;
1100
1101         ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1102                                     NILFS_BTREE_LEVEL_NODE_MIN);
1103         if (ret != -ENOENT) {
1104                 if (ret == 0)
1105                         ret = -EEXIST;
1106                 goto out;
1107         }
1108
1109         ret = nilfs_btree_prepare_insert(btree, path, &level, key, ptr, &stats);
1110         if (ret < 0)
1111                 goto out;
1112         nilfs_btree_commit_insert(btree, path, level, key, ptr);
1113         nilfs_bmap_add_blocks(btree, stats.bs_nblocks);
1114
1115  out:
1116         nilfs_btree_free_path(path);
1117         return ret;
1118 }
1119
1120 static void nilfs_btree_do_delete(struct nilfs_bmap *btree,
1121                                   struct nilfs_btree_path *path,
1122                                   int level, __u64 *keyp, __u64 *ptrp)
1123 {
1124         struct nilfs_btree_node *node;
1125         int ncblk;
1126
1127         if (level < nilfs_btree_height(btree) - 1) {
1128                 node = nilfs_btree_get_nonroot_node(path, level);
1129                 ncblk = nilfs_btree_nchildren_per_block(btree);
1130                 nilfs_btree_node_delete(node, path[level].bp_index,
1131                                         keyp, ptrp, ncblk);
1132                 if (!buffer_dirty(path[level].bp_bh))
1133                         nilfs_btnode_mark_dirty(path[level].bp_bh);
1134                 if (path[level].bp_index == 0)
1135                         nilfs_btree_promote_key(btree, path, level + 1,
1136                                 nilfs_btree_node_get_key(node, 0));
1137         } else {
1138                 node = nilfs_btree_get_root(btree);
1139                 nilfs_btree_node_delete(node, path[level].bp_index,
1140                                         keyp, ptrp,
1141                                         NILFS_BTREE_ROOT_NCHILDREN_MAX);
1142         }
1143 }
1144
1145 static void nilfs_btree_borrow_left(struct nilfs_bmap *btree,
1146                                     struct nilfs_btree_path *path,
1147                                     int level, __u64 *keyp, __u64 *ptrp)
1148 {
1149         struct nilfs_btree_node *node, *left;
1150         int nchildren, lnchildren, n, ncblk;
1151
1152         nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1153
1154         node = nilfs_btree_get_nonroot_node(path, level);
1155         left = nilfs_btree_get_sib_node(path, level);
1156         nchildren = nilfs_btree_node_get_nchildren(node);
1157         lnchildren = nilfs_btree_node_get_nchildren(left);
1158         ncblk = nilfs_btree_nchildren_per_block(btree);
1159
1160         n = (nchildren + lnchildren) / 2 - nchildren;
1161
1162         nilfs_btree_node_move_right(left, node, n, ncblk, ncblk);
1163
1164         if (!buffer_dirty(path[level].bp_bh))
1165                 nilfs_btnode_mark_dirty(path[level].bp_bh);
1166         if (!buffer_dirty(path[level].bp_sib_bh))
1167                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1168
1169         nilfs_btree_promote_key(btree, path, level + 1,
1170                                 nilfs_btree_node_get_key(node, 0));
1171
1172         brelse(path[level].bp_sib_bh);
1173         path[level].bp_sib_bh = NULL;
1174         path[level].bp_index += n;
1175 }
1176
1177 static void nilfs_btree_borrow_right(struct nilfs_bmap *btree,
1178                                      struct nilfs_btree_path *path,
1179                                      int level, __u64 *keyp, __u64 *ptrp)
1180 {
1181         struct nilfs_btree_node *node, *right;
1182         int nchildren, rnchildren, n, ncblk;
1183
1184         nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1185
1186         node = nilfs_btree_get_nonroot_node(path, level);
1187         right = nilfs_btree_get_sib_node(path, level);
1188         nchildren = nilfs_btree_node_get_nchildren(node);
1189         rnchildren = nilfs_btree_node_get_nchildren(right);
1190         ncblk = nilfs_btree_nchildren_per_block(btree);
1191
1192         n = (nchildren + rnchildren) / 2 - nchildren;
1193
1194         nilfs_btree_node_move_left(node, right, n, ncblk, ncblk);
1195
1196         if (!buffer_dirty(path[level].bp_bh))
1197                 nilfs_btnode_mark_dirty(path[level].bp_bh);
1198         if (!buffer_dirty(path[level].bp_sib_bh))
1199                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1200
1201         path[level + 1].bp_index++;
1202         nilfs_btree_promote_key(btree, path, level + 1,
1203                                 nilfs_btree_node_get_key(right, 0));
1204         path[level + 1].bp_index--;
1205
1206         brelse(path[level].bp_sib_bh);
1207         path[level].bp_sib_bh = NULL;
1208 }
1209
1210 static void nilfs_btree_concat_left(struct nilfs_bmap *btree,
1211                                     struct nilfs_btree_path *path,
1212                                     int level, __u64 *keyp, __u64 *ptrp)
1213 {
1214         struct nilfs_btree_node *node, *left;
1215         int n, ncblk;
1216
1217         nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1218
1219         node = nilfs_btree_get_nonroot_node(path, level);
1220         left = nilfs_btree_get_sib_node(path, level);
1221         ncblk = nilfs_btree_nchildren_per_block(btree);
1222
1223         n = nilfs_btree_node_get_nchildren(node);
1224
1225         nilfs_btree_node_move_left(left, node, n, ncblk, ncblk);
1226
1227         if (!buffer_dirty(path[level].bp_sib_bh))
1228                 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1229
1230         nilfs_btnode_delete(path[level].bp_bh);
1231         path[level].bp_bh = path[level].bp_sib_bh;
1232         path[level].bp_sib_bh = NULL;
1233         path[level].bp_index += nilfs_btree_node_get_nchildren(left);
1234 }
1235
1236 static void nilfs_btree_concat_right(struct nilfs_bmap *btree,
1237                                      struct nilfs_btree_path *path,
1238                                      int level, __u64 *keyp, __u64 *ptrp)
1239 {
1240         struct nilfs_btree_node *node, *right;
1241         int n, ncblk;
1242
1243         nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1244
1245         node = nilfs_btree_get_nonroot_node(path, level);
1246         right = nilfs_btree_get_sib_node(path, level);
1247         ncblk = nilfs_btree_nchildren_per_block(btree);
1248
1249         n = nilfs_btree_node_get_nchildren(right);
1250
1251         nilfs_btree_node_move_left(node, right, n, ncblk, ncblk);
1252
1253         if (!buffer_dirty(path[level].bp_bh))
1254                 nilfs_btnode_mark_dirty(path[level].bp_bh);
1255
1256         nilfs_btnode_delete(path[level].bp_sib_bh);
1257         path[level].bp_sib_bh = NULL;
1258         path[level + 1].bp_index++;
1259 }
1260
1261 static void nilfs_btree_shrink(struct nilfs_bmap *btree,
1262                                struct nilfs_btree_path *path,
1263                                int level, __u64 *keyp, __u64 *ptrp)
1264 {
1265         struct nilfs_btree_node *root, *child;
1266         int n, ncblk;
1267
1268         nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1269
1270         root = nilfs_btree_get_root(btree);
1271         child = nilfs_btree_get_nonroot_node(path, level);
1272         ncblk = nilfs_btree_nchildren_per_block(btree);
1273
1274         nilfs_btree_node_delete(root, 0, NULL, NULL,
1275                                 NILFS_BTREE_ROOT_NCHILDREN_MAX);
1276         nilfs_btree_node_set_level(root, level);
1277         n = nilfs_btree_node_get_nchildren(child);
1278         nilfs_btree_node_move_left(root, child, n,
1279                                    NILFS_BTREE_ROOT_NCHILDREN_MAX, ncblk);
1280
1281         nilfs_btnode_delete(path[level].bp_bh);
1282         path[level].bp_bh = NULL;
1283 }
1284
1285
1286 static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree,
1287                                       struct nilfs_btree_path *path,
1288                                       int *levelp,
1289                                       struct nilfs_bmap_stats *stats,
1290                                       struct inode *dat)
1291 {
1292         struct buffer_head *bh;
1293         struct nilfs_btree_node *node, *parent, *sib;
1294         __u64 sibptr;
1295         int pindex, level, ncmin, ncmax, ncblk, ret;
1296
1297         ret = 0;
1298         stats->bs_nblocks = 0;
1299         ncmin = NILFS_BTREE_NODE_NCHILDREN_MIN(nilfs_btree_node_size(btree));
1300         ncblk = nilfs_btree_nchildren_per_block(btree);
1301
1302         for (level = NILFS_BTREE_LEVEL_NODE_MIN;
1303              level < nilfs_btree_height(btree) - 1;
1304              level++) {
1305                 node = nilfs_btree_get_nonroot_node(path, level);
1306                 path[level].bp_oldreq.bpr_ptr =
1307                         nilfs_btree_node_get_ptr(node, path[level].bp_index,
1308                                                  ncblk);
1309                 ret = nilfs_bmap_prepare_end_ptr(btree,
1310                                                  &path[level].bp_oldreq, dat);
1311                 if (ret < 0)
1312                         goto err_out_child_node;
1313
1314                 if (nilfs_btree_node_get_nchildren(node) > ncmin) {
1315                         path[level].bp_op = nilfs_btree_do_delete;
1316                         stats->bs_nblocks++;
1317                         goto out;
1318                 }
1319
1320                 parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
1321                 pindex = path[level + 1].bp_index;
1322
1323                 if (pindex > 0) {
1324                         /* left sibling */
1325                         sibptr = nilfs_btree_node_get_ptr(parent, pindex - 1,
1326                                                           ncmax);
1327                         ret = nilfs_btree_get_block(btree, sibptr, &bh);
1328                         if (ret < 0)
1329                                 goto err_out_curr_node;
1330                         sib = (struct nilfs_btree_node *)bh->b_data;
1331                         if (nilfs_btree_node_get_nchildren(sib) > ncmin) {
1332                                 path[level].bp_sib_bh = bh;
1333                                 path[level].bp_op = nilfs_btree_borrow_left;
1334                                 stats->bs_nblocks++;
1335                                 goto out;
1336                         } else {
1337                                 path[level].bp_sib_bh = bh;
1338                                 path[level].bp_op = nilfs_btree_concat_left;
1339                                 stats->bs_nblocks++;
1340                                 /* continue; */
1341                         }
1342                 } else if (pindex <
1343                            nilfs_btree_node_get_nchildren(parent) - 1) {
1344                         /* right sibling */
1345                         sibptr = nilfs_btree_node_get_ptr(parent, pindex + 1,
1346                                                           ncmax);
1347                         ret = nilfs_btree_get_block(btree, sibptr, &bh);
1348                         if (ret < 0)
1349                                 goto err_out_curr_node;
1350                         sib = (struct nilfs_btree_node *)bh->b_data;
1351                         if (nilfs_btree_node_get_nchildren(sib) > ncmin) {
1352                                 path[level].bp_sib_bh = bh;
1353                                 path[level].bp_op = nilfs_btree_borrow_right;
1354                                 stats->bs_nblocks++;
1355                                 goto out;
1356                         } else {
1357                                 path[level].bp_sib_bh = bh;
1358                                 path[level].bp_op = nilfs_btree_concat_right;
1359                                 stats->bs_nblocks++;
1360                                 /* continue; */
1361                         }
1362                 } else {
1363                         /* no siblings */
1364                         /* the only child of the root node */
1365                         WARN_ON(level != nilfs_btree_height(btree) - 2);
1366                         if (nilfs_btree_node_get_nchildren(node) - 1 <=
1367                             NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1368                                 path[level].bp_op = nilfs_btree_shrink;
1369                                 stats->bs_nblocks += 2;
1370                         } else {
1371                                 path[level].bp_op = nilfs_btree_do_delete;
1372                                 stats->bs_nblocks++;
1373                         }
1374
1375                         goto out;
1376
1377                 }
1378         }
1379
1380         node = nilfs_btree_get_root(btree);
1381         path[level].bp_oldreq.bpr_ptr =
1382                 nilfs_btree_node_get_ptr(node, path[level].bp_index,
1383                                          NILFS_BTREE_ROOT_NCHILDREN_MAX);
1384
1385         ret = nilfs_bmap_prepare_end_ptr(btree, &path[level].bp_oldreq, dat);
1386         if (ret < 0)
1387                 goto err_out_child_node;
1388
1389         /* child of the root node is deleted */
1390         path[level].bp_op = nilfs_btree_do_delete;
1391         stats->bs_nblocks++;
1392
1393         /* success */
1394  out:
1395         *levelp = level;
1396         return ret;
1397
1398         /* error */
1399  err_out_curr_node:
1400         nilfs_bmap_abort_end_ptr(btree, &path[level].bp_oldreq, dat);
1401  err_out_child_node:
1402         for (level--; level >= NILFS_BTREE_LEVEL_NODE_MIN; level--) {
1403                 brelse(path[level].bp_sib_bh);
1404                 nilfs_bmap_abort_end_ptr(btree, &path[level].bp_oldreq, dat);
1405         }
1406         *levelp = level;
1407         stats->bs_nblocks = 0;
1408         return ret;
1409 }
1410
1411 static void nilfs_btree_commit_delete(struct nilfs_bmap *btree,
1412                                       struct nilfs_btree_path *path,
1413                                       int maxlevel, struct inode *dat)
1414 {
1415         int level;
1416
1417         for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1418                 nilfs_bmap_commit_end_ptr(btree, &path[level].bp_oldreq, dat);
1419                 path[level].bp_op(btree, path, level, NULL, NULL);
1420         }
1421
1422         if (!nilfs_bmap_dirty(btree))
1423                 nilfs_bmap_set_dirty(btree);
1424 }
1425
1426 static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key)
1427
1428 {
1429         struct nilfs_btree_path *path;
1430         struct nilfs_bmap_stats stats;
1431         struct inode *dat;
1432         int level, ret;
1433
1434         path = nilfs_btree_alloc_path();
1435         if (path == NULL)
1436                 return -ENOMEM;
1437
1438         ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1439                                     NILFS_BTREE_LEVEL_NODE_MIN);
1440         if (ret < 0)
1441                 goto out;
1442
1443
1444         dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL;
1445
1446         ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat);
1447         if (ret < 0)
1448                 goto out;
1449         nilfs_btree_commit_delete(btree, path, level, dat);
1450         nilfs_bmap_sub_blocks(btree, stats.bs_nblocks);
1451
1452 out:
1453         nilfs_btree_free_path(path);
1454         return ret;
1455 }
1456
1457 static int nilfs_btree_last_key(const struct nilfs_bmap *btree, __u64 *keyp)
1458 {
1459         struct nilfs_btree_path *path;
1460         int ret;
1461
1462         path = nilfs_btree_alloc_path();
1463         if (path == NULL)
1464                 return -ENOMEM;
1465
1466         ret = nilfs_btree_do_lookup_last(btree, path, keyp, NULL);
1467
1468         nilfs_btree_free_path(path);
1469
1470         return ret;
1471 }
1472
1473 static int nilfs_btree_check_delete(struct nilfs_bmap *btree, __u64 key)
1474 {
1475         struct buffer_head *bh;
1476         struct nilfs_btree_node *root, *node;
1477         __u64 maxkey, nextmaxkey;
1478         __u64 ptr;
1479         int nchildren, ret;
1480
1481         root = nilfs_btree_get_root(btree);
1482         switch (nilfs_btree_height(btree)) {
1483         case 2:
1484                 bh = NULL;
1485                 node = root;
1486                 break;
1487         case 3:
1488                 nchildren = nilfs_btree_node_get_nchildren(root);
1489                 if (nchildren > 1)
1490                         return 0;
1491                 ptr = nilfs_btree_node_get_ptr(root, nchildren - 1,
1492                                                NILFS_BTREE_ROOT_NCHILDREN_MAX);
1493                 ret = nilfs_btree_get_block(btree, ptr, &bh);
1494                 if (ret < 0)
1495                         return ret;
1496                 node = (struct nilfs_btree_node *)bh->b_data;
1497                 break;
1498         default:
1499                 return 0;
1500         }
1501
1502         nchildren = nilfs_btree_node_get_nchildren(node);
1503         maxkey = nilfs_btree_node_get_key(node, nchildren - 1);
1504         nextmaxkey = (nchildren > 1) ?
1505                 nilfs_btree_node_get_key(node, nchildren - 2) : 0;
1506         if (bh != NULL)
1507                 brelse(bh);
1508
1509         return (maxkey == key) && (nextmaxkey < NILFS_BMAP_LARGE_LOW);
1510 }
1511
1512 static int nilfs_btree_gather_data(struct nilfs_bmap *btree,
1513                                    __u64 *keys, __u64 *ptrs, int nitems)
1514 {
1515         struct buffer_head *bh;
1516         struct nilfs_btree_node *node, *root;
1517         __le64 *dkeys;
1518         __le64 *dptrs;
1519         __u64 ptr;
1520         int nchildren, ncmax, i, ret;
1521
1522         root = nilfs_btree_get_root(btree);
1523         switch (nilfs_btree_height(btree)) {
1524         case 2:
1525                 bh = NULL;
1526                 node = root;
1527                 ncmax = NILFS_BTREE_ROOT_NCHILDREN_MAX;
1528                 break;
1529         case 3:
1530                 nchildren = nilfs_btree_node_get_nchildren(root);
1531                 WARN_ON(nchildren > 1);
1532                 ptr = nilfs_btree_node_get_ptr(root, nchildren - 1,
1533                                                NILFS_BTREE_ROOT_NCHILDREN_MAX);
1534                 ret = nilfs_btree_get_block(btree, ptr, &bh);
1535                 if (ret < 0)
1536                         return ret;
1537                 node = (struct nilfs_btree_node *)bh->b_data;
1538                 ncmax = nilfs_btree_nchildren_per_block(btree);
1539                 break;
1540         default:
1541                 node = NULL;
1542                 return -EINVAL;
1543         }
1544
1545         nchildren = nilfs_btree_node_get_nchildren(node);
1546         if (nchildren < nitems)
1547                 nitems = nchildren;
1548         dkeys = nilfs_btree_node_dkeys(node);
1549         dptrs = nilfs_btree_node_dptrs(node, ncmax);
1550         for (i = 0; i < nitems; i++) {
1551                 keys[i] = le64_to_cpu(dkeys[i]);
1552                 ptrs[i] = le64_to_cpu(dptrs[i]);
1553         }
1554
1555         if (bh != NULL)
1556                 brelse(bh);
1557
1558         return nitems;
1559 }
1560
1561 static int
1562 nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key,
1563                                        union nilfs_bmap_ptr_req *dreq,
1564                                        union nilfs_bmap_ptr_req *nreq,
1565                                        struct buffer_head **bhp,
1566                                        struct nilfs_bmap_stats *stats)
1567 {
1568         struct buffer_head *bh;
1569         struct inode *dat = NULL;
1570         int ret;
1571
1572         stats->bs_nblocks = 0;
1573
1574         /* for data */
1575         /* cannot find near ptr */
1576         if (NILFS_BMAP_USE_VBN(btree)) {
1577                 dreq->bpr_ptr = nilfs_btree_find_target_v(btree, NULL, key);
1578                 dat = nilfs_bmap_get_dat(btree);
1579         }
1580
1581         ret = nilfs_bmap_prepare_alloc_ptr(btree, dreq, dat);
1582         if (ret < 0)
1583                 return ret;
1584
1585         *bhp = NULL;
1586         stats->bs_nblocks++;
1587         if (nreq != NULL) {
1588                 nreq->bpr_ptr = dreq->bpr_ptr + 1;
1589                 ret = nilfs_bmap_prepare_alloc_ptr(btree, nreq, dat);
1590                 if (ret < 0)
1591                         goto err_out_dreq;
1592
1593                 ret = nilfs_btree_get_new_block(btree, nreq->bpr_ptr, &bh);
1594                 if (ret < 0)
1595                         goto err_out_nreq;
1596
1597                 *bhp = bh;
1598                 stats->bs_nblocks++;
1599         }
1600
1601         /* success */
1602         return 0;
1603
1604         /* error */
1605  err_out_nreq:
1606         nilfs_bmap_abort_alloc_ptr(btree, nreq, dat);
1607  err_out_dreq:
1608         nilfs_bmap_abort_alloc_ptr(btree, dreq, dat);
1609         stats->bs_nblocks = 0;
1610         return ret;
1611
1612 }
1613
1614 static void
1615 nilfs_btree_commit_convert_and_insert(struct nilfs_bmap *btree,
1616                                       __u64 key, __u64 ptr,
1617                                       const __u64 *keys, const __u64 *ptrs,
1618                                       int n,
1619                                       union nilfs_bmap_ptr_req *dreq,
1620                                       union nilfs_bmap_ptr_req *nreq,
1621                                       struct buffer_head *bh)
1622 {
1623         struct nilfs_btree_node *node;
1624         struct inode *dat;
1625         __u64 tmpptr;
1626         int ncblk;
1627
1628         /* free resources */
1629         if (btree->b_ops->bop_clear != NULL)
1630                 btree->b_ops->bop_clear(btree);
1631
1632         /* ptr must be a pointer to a buffer head. */
1633         set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1634
1635         /* convert and insert */
1636         dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL;
1637         nilfs_btree_init(btree);
1638         if (nreq != NULL) {
1639                 nilfs_bmap_commit_alloc_ptr(btree, dreq, dat);
1640                 nilfs_bmap_commit_alloc_ptr(btree, nreq, dat);
1641
1642                 /* create child node at level 1 */
1643                 node = (struct nilfs_btree_node *)bh->b_data;
1644                 ncblk = nilfs_btree_nchildren_per_block(btree);
1645                 nilfs_btree_node_init(node, 0, 1, n, ncblk, keys, ptrs);
1646                 nilfs_btree_node_insert(node, n, key, dreq->bpr_ptr, ncblk);
1647                 if (!buffer_dirty(bh))
1648                         nilfs_btnode_mark_dirty(bh);
1649                 if (!nilfs_bmap_dirty(btree))
1650                         nilfs_bmap_set_dirty(btree);
1651
1652                 brelse(bh);
1653
1654                 /* create root node at level 2 */
1655                 node = nilfs_btree_get_root(btree);
1656                 tmpptr = nreq->bpr_ptr;
1657                 nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, 2, 1,
1658                                       NILFS_BTREE_ROOT_NCHILDREN_MAX,
1659                                       &keys[0], &tmpptr);
1660         } else {
1661                 nilfs_bmap_commit_alloc_ptr(btree, dreq, dat);
1662
1663                 /* create root node at level 1 */
1664                 node = nilfs_btree_get_root(btree);
1665                 nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, 1, n,
1666                                       NILFS_BTREE_ROOT_NCHILDREN_MAX,
1667                                       keys, ptrs);
1668                 nilfs_btree_node_insert(node, n, key, dreq->bpr_ptr,
1669                                         NILFS_BTREE_ROOT_NCHILDREN_MAX);
1670                 if (!nilfs_bmap_dirty(btree))
1671                         nilfs_bmap_set_dirty(btree);
1672         }
1673
1674         if (NILFS_BMAP_USE_VBN(btree))
1675                 nilfs_bmap_set_target_v(btree, key, dreq->bpr_ptr);
1676 }
1677
1678 /**
1679  * nilfs_btree_convert_and_insert -
1680  * @bmap:
1681  * @key:
1682  * @ptr:
1683  * @keys:
1684  * @ptrs:
1685  * @n:
1686  */
1687 int nilfs_btree_convert_and_insert(struct nilfs_bmap *btree,
1688                                    __u64 key, __u64 ptr,
1689                                    const __u64 *keys, const __u64 *ptrs, int n)
1690 {
1691         struct buffer_head *bh;
1692         union nilfs_bmap_ptr_req dreq, nreq, *di, *ni;
1693         struct nilfs_bmap_stats stats;
1694         int ret;
1695
1696         if (n + 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1697                 di = &dreq;
1698                 ni = NULL;
1699         } else if ((n + 1) <= NILFS_BTREE_NODE_NCHILDREN_MAX(
1700                            1 << btree->b_inode->i_blkbits)) {
1701                 di = &dreq;
1702                 ni = &nreq;
1703         } else {
1704                 di = NULL;
1705                 ni = NULL;
1706                 BUG();
1707         }
1708
1709         ret = nilfs_btree_prepare_convert_and_insert(btree, key, di, ni, &bh,
1710                                                      &stats);
1711         if (ret < 0)
1712                 return ret;
1713         nilfs_btree_commit_convert_and_insert(btree, key, ptr, keys, ptrs, n,
1714                                               di, ni, bh);
1715         nilfs_bmap_add_blocks(btree, stats.bs_nblocks);
1716         return 0;
1717 }
1718
1719 static int nilfs_btree_propagate_p(struct nilfs_bmap *btree,
1720                                    struct nilfs_btree_path *path,
1721                                    int level,
1722                                    struct buffer_head *bh)
1723 {
1724         while ((++level < nilfs_btree_height(btree) - 1) &&
1725                !buffer_dirty(path[level].bp_bh))
1726                 nilfs_btnode_mark_dirty(path[level].bp_bh);
1727
1728         return 0;
1729 }
1730
1731 static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree,
1732                                         struct nilfs_btree_path *path,
1733                                         int level, struct inode *dat)
1734 {
1735         struct nilfs_btree_node *parent;
1736         int ncmax, ret;
1737
1738         parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
1739         path[level].bp_oldreq.bpr_ptr =
1740                 nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index,
1741                                          ncmax);
1742         path[level].bp_newreq.bpr_ptr = path[level].bp_oldreq.bpr_ptr + 1;
1743         ret = nilfs_dat_prepare_update(dat, &path[level].bp_oldreq.bpr_req,
1744                                        &path[level].bp_newreq.bpr_req);
1745         if (ret < 0)
1746                 return ret;
1747
1748         if (buffer_nilfs_node(path[level].bp_bh)) {
1749                 path[level].bp_ctxt.oldkey = path[level].bp_oldreq.bpr_ptr;
1750                 path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr;
1751                 path[level].bp_ctxt.bh = path[level].bp_bh;
1752                 ret = nilfs_btnode_prepare_change_key(
1753                         &NILFS_BMAP_I(btree)->i_btnode_cache,
1754                         &path[level].bp_ctxt);
1755                 if (ret < 0) {
1756                         nilfs_dat_abort_update(dat,
1757                                                &path[level].bp_oldreq.bpr_req,
1758                                                &path[level].bp_newreq.bpr_req);
1759                         return ret;
1760                 }
1761         }
1762
1763         return 0;
1764 }
1765
1766 static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree,
1767                                         struct nilfs_btree_path *path,
1768                                         int level, struct inode *dat)
1769 {
1770         struct nilfs_btree_node *parent;
1771         int ncmax;
1772
1773         nilfs_dat_commit_update(dat, &path[level].bp_oldreq.bpr_req,
1774                                 &path[level].bp_newreq.bpr_req,
1775                                 btree->b_ptr_type == NILFS_BMAP_PTR_VS);
1776
1777         if (buffer_nilfs_node(path[level].bp_bh)) {
1778                 nilfs_btnode_commit_change_key(
1779                         &NILFS_BMAP_I(btree)->i_btnode_cache,
1780                         &path[level].bp_ctxt);
1781                 path[level].bp_bh = path[level].bp_ctxt.bh;
1782         }
1783         set_buffer_nilfs_volatile(path[level].bp_bh);
1784
1785         parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
1786         nilfs_btree_node_set_ptr(parent, path[level + 1].bp_index,
1787                                  path[level].bp_newreq.bpr_ptr, ncmax);
1788 }
1789
1790 static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree,
1791                                        struct nilfs_btree_path *path,
1792                                        int level, struct inode *dat)
1793 {
1794         nilfs_dat_abort_update(dat, &path[level].bp_oldreq.bpr_req,
1795                                &path[level].bp_newreq.bpr_req);
1796         if (buffer_nilfs_node(path[level].bp_bh))
1797                 nilfs_btnode_abort_change_key(
1798                         &NILFS_BMAP_I(btree)->i_btnode_cache,
1799                         &path[level].bp_ctxt);
1800 }
1801
1802 static int nilfs_btree_prepare_propagate_v(struct nilfs_bmap *btree,
1803                                            struct nilfs_btree_path *path,
1804                                            int minlevel, int *maxlevelp,
1805                                            struct inode *dat)
1806 {
1807         int level, ret;
1808
1809         level = minlevel;
1810         if (!buffer_nilfs_volatile(path[level].bp_bh)) {
1811                 ret = nilfs_btree_prepare_update_v(btree, path, level, dat);
1812                 if (ret < 0)
1813                         return ret;
1814         }
1815         while ((++level < nilfs_btree_height(btree) - 1) &&
1816                !buffer_dirty(path[level].bp_bh)) {
1817
1818                 WARN_ON(buffer_nilfs_volatile(path[level].bp_bh));
1819                 ret = nilfs_btree_prepare_update_v(btree, path, level, dat);
1820                 if (ret < 0)
1821                         goto out;
1822         }
1823
1824         /* success */
1825         *maxlevelp = level - 1;
1826         return 0;
1827
1828         /* error */
1829  out:
1830         while (--level > minlevel)
1831                 nilfs_btree_abort_update_v(btree, path, level, dat);
1832         if (!buffer_nilfs_volatile(path[level].bp_bh))
1833                 nilfs_btree_abort_update_v(btree, path, level, dat);
1834         return ret;
1835 }
1836
1837 static void nilfs_btree_commit_propagate_v(struct nilfs_bmap *btree,
1838                                            struct nilfs_btree_path *path,
1839                                            int minlevel, int maxlevel,
1840                                            struct buffer_head *bh,
1841                                            struct inode *dat)
1842 {
1843         int level;
1844
1845         if (!buffer_nilfs_volatile(path[minlevel].bp_bh))
1846                 nilfs_btree_commit_update_v(btree, path, minlevel, dat);
1847
1848         for (level = minlevel + 1; level <= maxlevel; level++)
1849                 nilfs_btree_commit_update_v(btree, path, level, dat);
1850 }
1851
1852 static int nilfs_btree_propagate_v(struct nilfs_bmap *btree,
1853                                    struct nilfs_btree_path *path,
1854                                    int level, struct buffer_head *bh)
1855 {
1856         int maxlevel = 0, ret;
1857         struct nilfs_btree_node *parent;
1858         struct inode *dat = nilfs_bmap_get_dat(btree);
1859         __u64 ptr;
1860         int ncmax;
1861
1862         get_bh(bh);
1863         path[level].bp_bh = bh;
1864         ret = nilfs_btree_prepare_propagate_v(btree, path, level, &maxlevel,
1865                                               dat);
1866         if (ret < 0)
1867                 goto out;
1868
1869         if (buffer_nilfs_volatile(path[level].bp_bh)) {
1870                 parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
1871                 ptr = nilfs_btree_node_get_ptr(parent,
1872                                                path[level + 1].bp_index,
1873                                                ncmax);
1874                 ret = nilfs_dat_mark_dirty(dat, ptr);
1875                 if (ret < 0)
1876                         goto out;
1877         }
1878
1879         nilfs_btree_commit_propagate_v(btree, path, level, maxlevel, bh, dat);
1880
1881  out:
1882         brelse(path[level].bp_bh);
1883         path[level].bp_bh = NULL;
1884         return ret;
1885 }
1886
1887 static int nilfs_btree_propagate(struct nilfs_bmap *btree,
1888                                  struct buffer_head *bh)
1889 {
1890         struct nilfs_btree_path *path;
1891         struct nilfs_btree_node *node;
1892         __u64 key;
1893         int level, ret;
1894
1895         WARN_ON(!buffer_dirty(bh));
1896
1897         path = nilfs_btree_alloc_path();
1898         if (path == NULL)
1899                 return -ENOMEM;
1900
1901         if (buffer_nilfs_node(bh)) {
1902                 node = (struct nilfs_btree_node *)bh->b_data;
1903                 key = nilfs_btree_node_get_key(node, 0);
1904                 level = nilfs_btree_node_get_level(node);
1905         } else {
1906                 key = nilfs_bmap_data_get_key(btree, bh);
1907                 level = NILFS_BTREE_LEVEL_DATA;
1908         }
1909
1910         ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1);
1911         if (ret < 0) {
1912                 if (unlikely(ret == -ENOENT))
1913                         printk(KERN_CRIT "%s: key = %llu, level == %d\n",
1914                                __func__, (unsigned long long)key, level);
1915                 goto out;
1916         }
1917
1918         ret = NILFS_BMAP_USE_VBN(btree) ?
1919                 nilfs_btree_propagate_v(btree, path, level, bh) :
1920                 nilfs_btree_propagate_p(btree, path, level, bh);
1921
1922  out:
1923         nilfs_btree_free_path(path);
1924
1925         return ret;
1926 }
1927
1928 static int nilfs_btree_propagate_gc(struct nilfs_bmap *btree,
1929                                     struct buffer_head *bh)
1930 {
1931         return nilfs_dat_mark_dirty(nilfs_bmap_get_dat(btree), bh->b_blocknr);
1932 }
1933
1934 static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree,
1935                                          struct list_head *lists,
1936                                          struct buffer_head *bh)
1937 {
1938         struct list_head *head;
1939         struct buffer_head *cbh;
1940         struct nilfs_btree_node *node, *cnode;
1941         __u64 key, ckey;
1942         int level;
1943
1944         get_bh(bh);
1945         node = (struct nilfs_btree_node *)bh->b_data;
1946         key = nilfs_btree_node_get_key(node, 0);
1947         level = nilfs_btree_node_get_level(node);
1948         if (level < NILFS_BTREE_LEVEL_NODE_MIN ||
1949             level >= NILFS_BTREE_LEVEL_MAX) {
1950                 dump_stack();
1951                 printk(KERN_WARNING
1952                        "%s: invalid btree level: %d (key=%llu, ino=%lu, "
1953                        "blocknr=%llu)\n",
1954                        __func__, level, (unsigned long long)key,
1955                        NILFS_BMAP_I(btree)->vfs_inode.i_ino,
1956                        (unsigned long long)bh->b_blocknr);
1957                 return;
1958         }
1959
1960         list_for_each(head, &lists[level]) {
1961                 cbh = list_entry(head, struct buffer_head, b_assoc_buffers);
1962                 cnode = (struct nilfs_btree_node *)cbh->b_data;
1963                 ckey = nilfs_btree_node_get_key(cnode, 0);
1964                 if (key < ckey)
1965                         break;
1966         }
1967         list_add_tail(&bh->b_assoc_buffers, head);
1968 }
1969
1970 static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree,
1971                                              struct list_head *listp)
1972 {
1973         struct address_space *btcache = &NILFS_BMAP_I(btree)->i_btnode_cache;
1974         struct list_head lists[NILFS_BTREE_LEVEL_MAX];
1975         struct pagevec pvec;
1976         struct buffer_head *bh, *head;
1977         pgoff_t index = 0;
1978         int level, i;
1979
1980         for (level = NILFS_BTREE_LEVEL_NODE_MIN;
1981              level < NILFS_BTREE_LEVEL_MAX;
1982              level++)
1983                 INIT_LIST_HEAD(&lists[level]);
1984
1985         pagevec_init(&pvec, 0);
1986
1987         while (pagevec_lookup_tag(&pvec, btcache, &index, PAGECACHE_TAG_DIRTY,
1988                                   PAGEVEC_SIZE)) {
1989                 for (i = 0; i < pagevec_count(&pvec); i++) {
1990                         bh = head = page_buffers(pvec.pages[i]);
1991                         do {
1992                                 if (buffer_dirty(bh))
1993                                         nilfs_btree_add_dirty_buffer(btree,
1994                                                                      lists, bh);
1995                         } while ((bh = bh->b_this_page) != head);
1996                 }
1997                 pagevec_release(&pvec);
1998                 cond_resched();
1999         }
2000
2001         for (level = NILFS_BTREE_LEVEL_NODE_MIN;
2002              level < NILFS_BTREE_LEVEL_MAX;
2003              level++)
2004                 list_splice_tail(&lists[level], listp);
2005 }
2006
2007 static int nilfs_btree_assign_p(struct nilfs_bmap *btree,
2008                                 struct nilfs_btree_path *path,
2009                                 int level,
2010                                 struct buffer_head **bh,
2011                                 sector_t blocknr,
2012                                 union nilfs_binfo *binfo)
2013 {
2014         struct nilfs_btree_node *parent;
2015         __u64 key;
2016         __u64 ptr;
2017         int ncmax, ret;
2018
2019         parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
2020         ptr = nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index,
2021                                        ncmax);
2022         if (buffer_nilfs_node(*bh)) {
2023                 path[level].bp_ctxt.oldkey = ptr;
2024                 path[level].bp_ctxt.newkey = blocknr;
2025                 path[level].bp_ctxt.bh = *bh;
2026                 ret = nilfs_btnode_prepare_change_key(
2027                         &NILFS_BMAP_I(btree)->i_btnode_cache,
2028                         &path[level].bp_ctxt);
2029                 if (ret < 0)
2030                         return ret;
2031                 nilfs_btnode_commit_change_key(
2032                         &NILFS_BMAP_I(btree)->i_btnode_cache,
2033                         &path[level].bp_ctxt);
2034                 *bh = path[level].bp_ctxt.bh;
2035         }
2036
2037         nilfs_btree_node_set_ptr(parent, path[level + 1].bp_index, blocknr,
2038                                  ncmax);
2039
2040         key = nilfs_btree_node_get_key(parent, path[level + 1].bp_index);
2041         /* on-disk format */
2042         binfo->bi_dat.bi_blkoff = cpu_to_le64(key);
2043         binfo->bi_dat.bi_level = level;
2044
2045         return 0;
2046 }
2047
2048 static int nilfs_btree_assign_v(struct nilfs_bmap *btree,
2049                                 struct nilfs_btree_path *path,
2050                                 int level,
2051                                 struct buffer_head **bh,
2052                                 sector_t blocknr,
2053                                 union nilfs_binfo *binfo)
2054 {
2055         struct nilfs_btree_node *parent;
2056         struct inode *dat = nilfs_bmap_get_dat(btree);
2057         __u64 key;
2058         __u64 ptr;
2059         union nilfs_bmap_ptr_req req;
2060         int ncmax, ret;
2061
2062         parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax);
2063         ptr = nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index,
2064                                        ncmax);
2065         req.bpr_ptr = ptr;
2066         ret = nilfs_dat_prepare_start(dat, &req.bpr_req);
2067         if (ret < 0)
2068                 return ret;
2069         nilfs_dat_commit_start(dat, &req.bpr_req, blocknr);
2070
2071         key = nilfs_btree_node_get_key(parent, path[level + 1].bp_index);
2072         /* on-disk format */
2073         binfo->bi_v.bi_vblocknr = cpu_to_le64(ptr);
2074         binfo->bi_v.bi_blkoff = cpu_to_le64(key);
2075
2076         return 0;
2077 }
2078
2079 static int nilfs_btree_assign(struct nilfs_bmap *btree,
2080                               struct buffer_head **bh,
2081                               sector_t blocknr,
2082                               union nilfs_binfo *binfo)
2083 {
2084         struct nilfs_btree_path *path;
2085         struct nilfs_btree_node *node;
2086         __u64 key;
2087         int level, ret;
2088
2089         path = nilfs_btree_alloc_path();
2090         if (path == NULL)
2091                 return -ENOMEM;
2092
2093         if (buffer_nilfs_node(*bh)) {
2094                 node = (struct nilfs_btree_node *)(*bh)->b_data;
2095                 key = nilfs_btree_node_get_key(node, 0);
2096                 level = nilfs_btree_node_get_level(node);
2097         } else {
2098                 key = nilfs_bmap_data_get_key(btree, *bh);
2099                 level = NILFS_BTREE_LEVEL_DATA;
2100         }
2101
2102         ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1);
2103         if (ret < 0) {
2104                 WARN_ON(ret == -ENOENT);
2105                 goto out;
2106         }
2107
2108         ret = NILFS_BMAP_USE_VBN(btree) ?
2109                 nilfs_btree_assign_v(btree, path, level, bh, blocknr, binfo) :
2110                 nilfs_btree_assign_p(btree, path, level, bh, blocknr, binfo);
2111
2112  out:
2113         nilfs_btree_free_path(path);
2114
2115         return ret;
2116 }
2117
2118 static int nilfs_btree_assign_gc(struct nilfs_bmap *btree,
2119                                  struct buffer_head **bh,
2120                                  sector_t blocknr,
2121                                  union nilfs_binfo *binfo)
2122 {
2123         struct nilfs_btree_node *node;
2124         __u64 key;
2125         int ret;
2126
2127         ret = nilfs_dat_move(nilfs_bmap_get_dat(btree), (*bh)->b_blocknr,
2128                              blocknr);
2129         if (ret < 0)
2130                 return ret;
2131
2132         if (buffer_nilfs_node(*bh)) {
2133                 node = (struct nilfs_btree_node *)(*bh)->b_data;
2134                 key = nilfs_btree_node_get_key(node, 0);
2135         } else
2136                 key = nilfs_bmap_data_get_key(btree, *bh);
2137
2138         /* on-disk format */
2139         binfo->bi_v.bi_vblocknr = cpu_to_le64((*bh)->b_blocknr);
2140         binfo->bi_v.bi_blkoff = cpu_to_le64(key);
2141
2142         return 0;
2143 }
2144
2145 static int nilfs_btree_mark(struct nilfs_bmap *btree, __u64 key, int level)
2146 {
2147         struct buffer_head *bh;
2148         struct nilfs_btree_path *path;
2149         __u64 ptr;
2150         int ret;
2151
2152         path = nilfs_btree_alloc_path();
2153         if (path == NULL)
2154                 return -ENOMEM;
2155
2156         ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level + 1);
2157         if (ret < 0) {
2158                 WARN_ON(ret == -ENOENT);
2159                 goto out;
2160         }
2161         ret = nilfs_btree_get_block(btree, ptr, &bh);
2162         if (ret < 0) {
2163                 WARN_ON(ret == -ENOENT);
2164                 goto out;
2165         }
2166
2167         if (!buffer_dirty(bh))
2168                 nilfs_btnode_mark_dirty(bh);
2169         brelse(bh);
2170         if (!nilfs_bmap_dirty(btree))
2171                 nilfs_bmap_set_dirty(btree);
2172
2173  out:
2174         nilfs_btree_free_path(path);
2175         return ret;
2176 }
2177
2178 static const struct nilfs_bmap_operations nilfs_btree_ops = {
2179         .bop_lookup             =       nilfs_btree_lookup,
2180         .bop_lookup_contig      =       nilfs_btree_lookup_contig,
2181         .bop_insert             =       nilfs_btree_insert,
2182         .bop_delete             =       nilfs_btree_delete,
2183         .bop_clear              =       NULL,
2184
2185         .bop_propagate          =       nilfs_btree_propagate,
2186
2187         .bop_lookup_dirty_buffers =     nilfs_btree_lookup_dirty_buffers,
2188
2189         .bop_assign             =       nilfs_btree_assign,
2190         .bop_mark               =       nilfs_btree_mark,
2191
2192         .bop_last_key           =       nilfs_btree_last_key,
2193         .bop_check_insert       =       NULL,
2194         .bop_check_delete       =       nilfs_btree_check_delete,
2195         .bop_gather_data        =       nilfs_btree_gather_data,
2196 };
2197
2198 static const struct nilfs_bmap_operations nilfs_btree_ops_gc = {
2199         .bop_lookup             =       NULL,
2200         .bop_lookup_contig      =       NULL,
2201         .bop_insert             =       NULL,
2202         .bop_delete             =       NULL,
2203         .bop_clear              =       NULL,
2204
2205         .bop_propagate          =       nilfs_btree_propagate_gc,
2206
2207         .bop_lookup_dirty_buffers =     nilfs_btree_lookup_dirty_buffers,
2208
2209         .bop_assign             =       nilfs_btree_assign_gc,
2210         .bop_mark               =       NULL,
2211
2212         .bop_last_key           =       NULL,
2213         .bop_check_insert       =       NULL,
2214         .bop_check_delete       =       NULL,
2215         .bop_gather_data        =       NULL,
2216 };
2217
2218 int nilfs_btree_init(struct nilfs_bmap *bmap)
2219 {
2220         bmap->b_ops = &nilfs_btree_ops;
2221         bmap->b_nchildren_per_block =
2222                 NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap));
2223         return 0;
2224 }
2225
2226 void nilfs_btree_init_gc(struct nilfs_bmap *bmap)
2227 {
2228         bmap->b_ops = &nilfs_btree_ops_gc;
2229         bmap->b_nchildren_per_block =
2230                 NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap));
2231 }