fs/f2fs/segment.c

   1 /*
   2  * fs/f2fs/segment.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/bio.h>
  14 #include <linux/blkdev.h>
  15 #include <linux/prefetch.h>
  16 #include <linux/kthread.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/swap.h>
  19
  20 #include "f2fs.h"
  21 #include "segment.h"
  22 #include "node.h"
  23 #include <trace/events/f2fs.h>
  24
  25 #define __reverse_ffz(x) __reverse_ffs(~(x))
  26
  27 static struct kmem_cache *discard_entry_slab;
  28 static struct kmem_cache *flush_cmd_slab;
  29
  30 /*
  31  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  32  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  33  */
  34 static inline unsigned long __reverse_ffs(unsigned long word)
  35 {
  36         int num = 0;
  37
  38 #if BITS_PER_LONG == 64
  39         if ((word & 0xffffffff) == 0) {
  40                 num += 32;
  41                 word >>= 32;
  42         }
  43 #endif
  44         if ((word & 0xffff) == 0) {
  45                 num += 16;
  46                 word >>= 16;
  47         }
  48         if ((word & 0xff) == 0) {
  49                 num += 8;
  50                 word >>= 8;
  51         }
  52         if ((word & 0xf0) == 0)
  53                 num += 4;
  54         else
  55                 word >>= 4;
  56         if ((word & 0xc) == 0)
  57                 num += 2;
  58         else
  59                 word >>= 2;
  60         if ((word & 0x2) == 0)
  61                 num += 1;
  62         return num;
  63 }
  64
  65 /*
  66  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c becasue
  67  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  68  * Example:
  69  *                             LSB <--> MSB
  70  *   f2fs_set_bit(0, bitmap) => 0000 0001
  71  *   f2fs_set_bit(7, bitmap) => 1000 0000
  72  */
  73 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  74                         unsigned long size, unsigned long offset)
  75 {
  76         const unsigned long *p = addr + BIT_WORD(offset);
  77         unsigned long result = offset & ~(BITS_PER_LONG - 1);
  78         unsigned long tmp;
  79         unsigned long mask, submask;
  80         unsigned long quot, rest;
  81
  82         if (offset >= size)
  83                 return size;
  84
  85         size -= result;
  86         offset %= BITS_PER_LONG;
  87         if (!offset)
  88                 goto aligned;
  89
  90         tmp = *(p++);
  91         quot = (offset >> 3) << 3;
  92         rest = offset & 0x7;
  93         mask = ~0UL << quot;
  94         submask = (unsigned char)(0xff << rest) >> rest;
  95         submask <<= quot;
  96         mask &= submask;
  97         tmp &= mask;
  98         if (size < BITS_PER_LONG)
  99                 goto found_first;
 100         if (tmp)
 101                 goto found_middle;
 102
 103         size -= BITS_PER_LONG;
 104         result += BITS_PER_LONG;
 105 aligned:
 106         while (size & ~(BITS_PER_LONG-1)) {
 107                 tmp = *(p++);
 108                 if (tmp)
 109                         goto found_middle;
 110                 result += BITS_PER_LONG;
 111                 size -= BITS_PER_LONG;
 112         }
 113         if (!size)
 114                 return result;
 115         tmp = *p;
 116 found_first:
 117         tmp &= (~0UL >> (BITS_PER_LONG - size));
 118         if (tmp == 0UL)         /* Are any bits set? */
 119                 return result + size;   /* Nope. */
 120 found_middle:
 121         return result + __reverse_ffs(tmp);
 122 }
 123
 124 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 125                         unsigned long size, unsigned long offset)
 126 {
 127         const unsigned long *p = addr + BIT_WORD(offset);
 128         unsigned long result = offset & ~(BITS_PER_LONG - 1);
 129         unsigned long tmp;
 130         unsigned long mask, submask;
 131         unsigned long quot, rest;
 132
 133         if (offset >= size)
 134                 return size;
 135
 136         size -= result;
 137         offset %= BITS_PER_LONG;
 138         if (!offset)
 139                 goto aligned;
 140
 141         tmp = *(p++);
 142         quot = (offset >> 3) << 3;
 143         rest = offset & 0x7;
 144         mask = ~(~0UL << quot);
 145         submask = (unsigned char)~((unsigned char)(0xff << rest) >> rest);
 146         submask <<= quot;
 147         mask += submask;
 148         tmp |= mask;
 149         if (size < BITS_PER_LONG)
 150                 goto found_first;
 151         if (~tmp)
 152                 goto found_middle;
 153
 154         size -= BITS_PER_LONG;
 155         result += BITS_PER_LONG;
 156 aligned:
 157         while (size & ~(BITS_PER_LONG - 1)) {
 158                 tmp = *(p++);
 159                 if (~tmp)
 160                         goto found_middle;
 161                 result += BITS_PER_LONG;
 162                 size -= BITS_PER_LONG;
 163         }
 164         if (!size)
 165                 return result;
 166         tmp = *p;
 167
 168 found_first:
 169         tmp |= ~0UL << size;
 170         if (tmp == ~0UL)        /* Are any bits zero? */
 171                 return result + size;   /* Nope. */
 172 found_middle:
 173         return result + __reverse_ffz(tmp);
 174 }
 175
 176 /*
 177  * This function balances dirty node and dentry pages.
 178  * In addition, it controls garbage collection.
 179  */
 180 void f2fs_balance_fs(struct f2fs_sb_info *sbi)
 181 {
 182         /*
 183          * We should do GC or end up with checkpoint, if there are so many dirty
 184          * dir/node pages without enough free segments.
 185          */
 186         if (has_not_enough_free_secs(sbi, 0)) {
 187                 mutex_lock(&sbi->gc_mutex);
 188                 f2fs_gc(sbi);
 189         }
 190 }
 191
 192 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 193 {
 194         /* check the # of cached NAT entries and prefree segments */
 195         if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) ||
 196                                 excess_prefree_segs(sbi))
 197                 f2fs_sync_fs(sbi->sb, true);
 198 }
 199
 200 static int issue_flush_thread(void *data)
 201 {
 202         struct f2fs_sb_info *sbi = data;
 203         struct f2fs_sm_info *sm_i = SM_I(sbi);
 204         wait_queue_head_t *q = &sm_i->flush_wait_queue;
 205 repeat:
 206         if (kthread_should_stop())
 207                 return 0;
 208
 209         spin_lock(&sm_i->issue_lock);
 210         if (sm_i->issue_list) {
 211                 sm_i->dispatch_list = sm_i->issue_list;
 212                 sm_i->issue_list = sm_i->issue_tail = NULL;
 213         }
 214         spin_unlock(&sm_i->issue_lock);
 215
 216         if (sm_i->dispatch_list) {
 217                 struct bio *bio = bio_alloc(GFP_NOIO, 0);
 218                 struct flush_cmd *cmd, *next;
 219                 int ret;
 220
 221                 bio->bi_bdev = sbi->sb->s_bdev;
 222                 ret = submit_bio_wait(WRITE_FLUSH, bio);
 223
 224                 for (cmd = sm_i->dispatch_list; cmd; cmd = next) {
 225                         cmd->ret = ret;
 226                         next = cmd->next;
 227                         complete(&cmd->wait);
 228                 }
 229                 sm_i->dispatch_list = NULL;
 230         }
 231
 232         wait_event_interruptible(*q, kthread_should_stop() || sm_i->issue_list);
 233         goto repeat;
 234 }
 235
 236 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
 237 {
 238         struct f2fs_sm_info *sm_i = SM_I(sbi);
 239         struct flush_cmd *cmd;
 240         int ret;
 241
 242         if (!test_opt(sbi, FLUSH_MERGE))
 243                 return blkdev_issue_flush(sbi->sb->s_bdev, GFP_KERNEL, NULL);
 244
 245         cmd = f2fs_kmem_cache_alloc(flush_cmd_slab, GFP_ATOMIC);
 246         cmd->next = NULL;
 247         cmd->ret = 0;
 248         init_completion(&cmd->wait);
 249
 250         spin_lock(&sm_i->issue_lock);
 251         if (sm_i->issue_list)
 252                 sm_i->issue_tail->next = cmd;
 253         else
 254                 sm_i->issue_list = cmd;
 255         sm_i->issue_tail = cmd;
 256         spin_unlock(&sm_i->issue_lock);
 257
 258         if (!sm_i->dispatch_list)
 259                 wake_up(&sm_i->flush_wait_queue);
 260
 261         wait_for_completion(&cmd->wait);
 262         ret = cmd->ret;
 263         kmem_cache_free(flush_cmd_slab, cmd);
 264         return ret;
 265 }
 266
 267 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 268                 enum dirty_type dirty_type)
 269 {
 270         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 271
 272         /* need not be added */
 273         if (IS_CURSEG(sbi, segno))
 274                 return;
 275
 276         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 277                 dirty_i->nr_dirty[dirty_type]++;
 278
 279         if (dirty_type == DIRTY) {
 280                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 281                 enum dirty_type t = sentry->type;
 282
 283                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 284                         dirty_i->nr_dirty[t]++;
 285         }
 286 }
 287
 288 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 289                 enum dirty_type dirty_type)
 290 {
 291         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 292
 293         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 294                 dirty_i->nr_dirty[dirty_type]--;
 295
 296         if (dirty_type == DIRTY) {
 297                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 298                 enum dirty_type t = sentry->type;
 299
 300                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 301                         dirty_i->nr_dirty[t]--;
 302
 303                 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
 304                         clear_bit(GET_SECNO(sbi, segno),
 305                                                 dirty_i->victim_secmap);
 306         }
 307 }
 308
 309 /*
 310  * Should not occur error such as -ENOMEM.
 311  * Adding dirty entry into seglist is not critical operation.
 312  * If a given segment is one of current working segments, it won't be added.
 313  */
 314 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 315 {
 316         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 317         unsigned short valid_blocks;
 318
 319         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 320                 return;
 321
 322         mutex_lock(&dirty_i->seglist_lock);
 323
 324         valid_blocks = get_valid_blocks(sbi, segno, 0);
 325
 326         if (valid_blocks == 0) {
 327                 __locate_dirty_segment(sbi, segno, PRE);
 328                 __remove_dirty_segment(sbi, segno, DIRTY);
 329         } else if (valid_blocks < sbi->blocks_per_seg) {
 330                 __locate_dirty_segment(sbi, segno, DIRTY);
 331         } else {
 332                 /* Recovery routine with SSR needs this */
 333                 __remove_dirty_segment(sbi, segno, DIRTY);
 334         }
 335
 336         mutex_unlock(&dirty_i->seglist_lock);
 337 }
 338
 339 static void f2fs_issue_discard(struct f2fs_sb_info *sbi,
 340                                 block_t blkstart, block_t blklen)
 341 {
 342         sector_t start = SECTOR_FROM_BLOCK(sbi, blkstart);
 343         sector_t len = SECTOR_FROM_BLOCK(sbi, blklen);
 344         blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
 345         trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
 346 }
 347
 348 static void add_discard_addrs(struct f2fs_sb_info *sbi,
 349                         unsigned int segno, struct seg_entry *se)
 350 {
 351         struct list_head *head = &SM_I(sbi)->discard_list;
 352         struct discard_entry *new;
 353         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
 354         int max_blocks = sbi->blocks_per_seg;
 355         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
 356         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
 357         unsigned long dmap[entries];
 358         unsigned int start = 0, end = -1;
 359         int i;
 360
 361         if (!test_opt(sbi, DISCARD))
 362                 return;
 363
 364         /* zero block will be discarded through the prefree list */
 365         if (!se->valid_blocks || se->valid_blocks == max_blocks)
 366                 return;
 367
 368         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
 369         for (i = 0; i < entries; i++)
 370                 dmap[i] = (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
 371
 372         while (SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
 373                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
 374                 if (start >= max_blocks)
 375                         break;
 376
 377                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
 378
 379                 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
 380                 INIT_LIST_HEAD(&new->list);
 381                 new->blkaddr = START_BLOCK(sbi, segno) + start;
 382                 new->len = end - start;
 383
 384                 list_add_tail(&new->list, head);
 385                 SM_I(sbi)->nr_discards += end - start;
 386         }
 387 }
 388
 389 /*
 390  * Should call clear_prefree_segments after checkpoint is done.
 391  */
 392 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
 393 {
 394         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 395         unsigned int segno = -1;
 396         unsigned int total_segs = TOTAL_SEGS(sbi);
 397
 398         mutex_lock(&dirty_i->seglist_lock);
 399         while (1) {
 400                 segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs,
 401                                 segno + 1);
 402                 if (segno >= total_segs)
 403                         break;
 404                 __set_test_and_free(sbi, segno);
 405         }
 406         mutex_unlock(&dirty_i->seglist_lock);
 407 }
 408
 409 void clear_prefree_segments(struct f2fs_sb_info *sbi)
 410 {
 411         struct list_head *head = &(SM_I(sbi)->discard_list);
 412         struct discard_entry *entry, *this;
 413         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 414         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
 415         unsigned int total_segs = TOTAL_SEGS(sbi);
 416         unsigned int start = 0, end = -1;
 417
 418         mutex_lock(&dirty_i->seglist_lock);
 419
 420         while (1) {
 421                 int i;
 422                 start = find_next_bit(prefree_map, total_segs, end + 1);
 423                 if (start >= total_segs)
 424                         break;
 425                 end = find_next_zero_bit(prefree_map, total_segs, start + 1);
 426
 427                 for (i = start; i < end; i++)
 428                         clear_bit(i, prefree_map);
 429
 430                 dirty_i->nr_dirty[PRE] -= end - start;
 431
 432                 if (!test_opt(sbi, DISCARD))
 433                         continue;
 434
 435                 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
 436                                 (end - start) << sbi->log_blocks_per_seg);
 437         }
 438         mutex_unlock(&dirty_i->seglist_lock);
 439
 440         /* send small discards */
 441         list_for_each_entry_safe(entry, this, head, list) {
 442                 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
 443                 list_del(&entry->list);
 444                 SM_I(sbi)->nr_discards -= entry->len;
 445                 kmem_cache_free(discard_entry_slab, entry);
 446         }
 447 }
 448
 449 static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
 450 {
 451         struct sit_info *sit_i = SIT_I(sbi);
 452         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap))
 453                 sit_i->dirty_sentries++;
 454 }
 455
 456 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
 457                                         unsigned int segno, int modified)
 458 {
 459         struct seg_entry *se = get_seg_entry(sbi, segno);
 460         se->type = type;
 461         if (modified)
 462                 __mark_sit_entry_dirty(sbi, segno);
 463 }
 464
 465 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
 466 {
 467         struct seg_entry *se;
 468         unsigned int segno, offset;
 469         long int new_vblocks;
 470
 471         segno = GET_SEGNO(sbi, blkaddr);
 472
 473         se = get_seg_entry(sbi, segno);
 474         new_vblocks = se->valid_blocks + del;
 475         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 476
 477         f2fs_bug_on((new_vblocks >> (sizeof(unsigned short) << 3) ||
 478                                 (new_vblocks > sbi->blocks_per_seg)));
 479
 480         se->valid_blocks = new_vblocks;
 481         se->mtime = get_mtime(sbi);
 482         SIT_I(sbi)->max_mtime = se->mtime;
 483
 484         /* Update valid block bitmap */
 485         if (del > 0) {
 486                 if (f2fs_set_bit(offset, se->cur_valid_map))
 487                         BUG();
 488         } else {
 489                 if (!f2fs_clear_bit(offset, se->cur_valid_map))
 490                         BUG();
 491         }
 492         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
 493                 se->ckpt_valid_blocks += del;
 494
 495         __mark_sit_entry_dirty(sbi, segno);
 496
 497         /* update total number of valid blocks to be written in ckpt area */
 498         SIT_I(sbi)->written_valid_blocks += del;
 499
 500         if (sbi->segs_per_sec > 1)
 501                 get_sec_entry(sbi, segno)->valid_blocks += del;
 502 }
 503
 504 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
 505 {
 506         update_sit_entry(sbi, new, 1);
 507         if (GET_SEGNO(sbi, old) != NULL_SEGNO)
 508                 update_sit_entry(sbi, old, -1);
 509
 510         locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
 511         locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
 512 }
 513
 514 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
 515 {
 516         unsigned int segno = GET_SEGNO(sbi, addr);
 517         struct sit_info *sit_i = SIT_I(sbi);
 518
 519         f2fs_bug_on(addr == NULL_ADDR);
 520         if (addr == NEW_ADDR)
 521                 return;
 522
 523         /* add it into sit main buffer */
 524         mutex_lock(&sit_i->sentry_lock);
 525
 526         update_sit_entry(sbi, addr, -1);
 527
 528         /* add it into dirty seglist */
 529         locate_dirty_segment(sbi, segno);
 530
 531         mutex_unlock(&sit_i->sentry_lock);
 532 }
 533
 534 /*
 535  * This function should be resided under the curseg_mutex lock
 536  */
 537 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
 538                                         struct f2fs_summary *sum)
 539 {
 540         struct curseg_info *curseg = CURSEG_I(sbi, type);
 541         void *addr = curseg->sum_blk;
 542         addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
 543         memcpy(addr, sum, sizeof(struct f2fs_summary));
 544 }
 545
 546 /*
 547  * Calculate the number of current summary pages for writing
 548  */
 549 int npages_for_summary_flush(struct f2fs_sb_info *sbi)
 550 {
 551         int valid_sum_count = 0;
 552         int i, sum_in_page;
 553
 554         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 555                 if (sbi->ckpt->alloc_type[i] == SSR)
 556                         valid_sum_count += sbi->blocks_per_seg;
 557                 else
 558                         valid_sum_count += curseg_blkoff(sbi, i);
 559         }
 560
 561         sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
 562                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
 563         if (valid_sum_count <= sum_in_page)
 564                 return 1;
 565         else if ((valid_sum_count - sum_in_page) <=
 566                 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
 567                 return 2;
 568         return 3;
 569 }
 570
 571 /*
 572  * Caller should put this summary page
 573  */
 574 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
 575 {
 576         return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
 577 }
 578
 579 static void write_sum_page(struct f2fs_sb_info *sbi,
 580                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
 581 {
 582         struct page *page = grab_meta_page(sbi, blk_addr);
 583         void *kaddr = page_address(page);
 584         memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE);
 585         set_page_dirty(page);
 586         f2fs_put_page(page, 1);
 587 }
 588
 589 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
 590 {
 591         struct curseg_info *curseg = CURSEG_I(sbi, type);
 592         unsigned int segno = curseg->segno + 1;
 593         struct free_segmap_info *free_i = FREE_I(sbi);
 594
 595         if (segno < TOTAL_SEGS(sbi) && segno % sbi->segs_per_sec)
 596                 return !test_bit(segno, free_i->free_segmap);
 597         return 0;
 598 }
 599
 600 /*
 601  * Find a new segment from the free segments bitmap to right order
 602  * This function should be returned with success, otherwise BUG
 603  */
 604 static void get_new_segment(struct f2fs_sb_info *sbi,
 605                         unsigned int *newseg, bool new_sec, int dir)
 606 {
 607         struct free_segmap_info *free_i = FREE_I(sbi);
 608         unsigned int segno, secno, zoneno;
 609         unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone;
 610         unsigned int hint = *newseg / sbi->segs_per_sec;
 611         unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
 612         unsigned int left_start = hint;
 613         bool init = true;
 614         int go_left = 0;
 615         int i;
 616
 617         write_lock(&free_i->segmap_lock);
 618
 619         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
 620                 segno = find_next_zero_bit(free_i->free_segmap,
 621                                         TOTAL_SEGS(sbi), *newseg + 1);
 622                 if (segno - *newseg < sbi->segs_per_sec -
 623                                         (*newseg % sbi->segs_per_sec))
 624                         goto got_it;
 625         }
 626 find_other_zone:
 627         secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint);
 628         if (secno >= TOTAL_SECS(sbi)) {
 629                 if (dir == ALLOC_RIGHT) {
 630                         secno = find_next_zero_bit(free_i->free_secmap,
 631                                                         TOTAL_SECS(sbi), 0);
 632                         f2fs_bug_on(secno >= TOTAL_SECS(sbi));
 633                 } else {
 634                         go_left = 1;
 635                         left_start = hint - 1;
 636                 }
 637         }
 638         if (go_left == 0)
 639                 goto skip_left;
 640
 641         while (test_bit(left_start, free_i->free_secmap)) {
 642                 if (left_start > 0) {
 643                         left_start--;
 644                         continue;
 645                 }
 646                 left_start = find_next_zero_bit(free_i->free_secmap,
 647                                                         TOTAL_SECS(sbi), 0);
 648                 f2fs_bug_on(left_start >= TOTAL_SECS(sbi));
 649                 break;
 650         }
 651         secno = left_start;
 652 skip_left:
 653         hint = secno;
 654         segno = secno * sbi->segs_per_sec;
 655         zoneno = secno / sbi->secs_per_zone;
 656
 657         /* give up on finding another zone */
 658         if (!init)
 659                 goto got_it;
 660         if (sbi->secs_per_zone == 1)
 661                 goto got_it;
 662         if (zoneno == old_zoneno)
 663                 goto got_it;
 664         if (dir == ALLOC_LEFT) {
 665                 if (!go_left && zoneno + 1 >= total_zones)
 666                         goto got_it;
 667                 if (go_left && zoneno == 0)
 668                         goto got_it;
 669         }
 670         for (i = 0; i < NR_CURSEG_TYPE; i++)
 671                 if (CURSEG_I(sbi, i)->zone == zoneno)
 672                         break;
 673
 674         if (i < NR_CURSEG_TYPE) {
 675                 /* zone is in user, try another */
 676                 if (go_left)
 677                         hint = zoneno * sbi->secs_per_zone - 1;
 678                 else if (zoneno + 1 >= total_zones)
 679                         hint = 0;
 680                 else
 681                         hint = (zoneno + 1) * sbi->secs_per_zone;
 682                 init = false;
 683                 goto find_other_zone;
 684         }
 685 got_it:
 686         /* set it as dirty segment in free segmap */
 687         f2fs_bug_on(test_bit(segno, free_i->free_segmap));
 688         __set_inuse(sbi, segno);
 689         *newseg = segno;
 690         write_unlock(&free_i->segmap_lock);
 691 }
 692
 693 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
 694 {
 695         struct curseg_info *curseg = CURSEG_I(sbi, type);
 696         struct summary_footer *sum_footer;
 697
 698         curseg->segno = curseg->next_segno;
 699         curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
 700         curseg->next_blkoff = 0;
 701         curseg->next_segno = NULL_SEGNO;
 702
 703         sum_footer = &(curseg->sum_blk->footer);
 704         memset(sum_footer, 0, sizeof(struct summary_footer));
 705         if (IS_DATASEG(type))
 706                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
 707         if (IS_NODESEG(type))
 708                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
 709         __set_sit_entry_type(sbi, type, curseg->segno, modified);
 710 }
 711
 712 /*
 713  * Allocate a current working segment.
 714  * This function always allocates a free segment in LFS manner.
 715  */
 716 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
 717 {
 718         struct curseg_info *curseg = CURSEG_I(sbi, type);
 719         unsigned int segno = curseg->segno;
 720         int dir = ALLOC_LEFT;
 721
 722         write_sum_page(sbi, curseg->sum_blk,
 723                                 GET_SUM_BLOCK(sbi, segno));
 724         if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
 725                 dir = ALLOC_RIGHT;
 726
 727         if (test_opt(sbi, NOHEAP))
 728                 dir = ALLOC_RIGHT;
 729
 730         get_new_segment(sbi, &segno, new_sec, dir);
 731         curseg->next_segno = segno;
 732         reset_curseg(sbi, type, 1);
 733         curseg->alloc_type = LFS;
 734 }
 735
 736 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
 737                         struct curseg_info *seg, block_t start)
 738 {
 739         struct seg_entry *se = get_seg_entry(sbi, seg->segno);
 740         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
 741         unsigned long target_map[entries];
 742         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
 743         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
 744         int i, pos;
 745
 746         for (i = 0; i < entries; i++)
 747                 target_map[i] = ckpt_map[i] | cur_map[i];
 748
 749         pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
 750
 751         seg->next_blkoff = pos;
 752 }
 753
 754 /*
 755  * If a segment is written by LFS manner, next block offset is just obtained
 756  * by increasing the current block offset. However, if a segment is written by
 757  * SSR manner, next block offset obtained by calling __next_free_blkoff
 758  */
 759 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
 760                                 struct curseg_info *seg)
 761 {
 762         if (seg->alloc_type == SSR)
 763                 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
 764         else
 765                 seg->next_blkoff++;
 766 }
 767
 768 /*
 769  * This function always allocates a used segment (from dirty seglist) by SSR
 770  * manner, so it should recover the existing segment information of valid blocks
 771  */
 772 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
 773 {
 774         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 775         struct curseg_info *curseg = CURSEG_I(sbi, type);
 776         unsigned int new_segno = curseg->next_segno;
 777         struct f2fs_summary_block *sum_node;
 778         struct page *sum_page;
 779
 780         write_sum_page(sbi, curseg->sum_blk,
 781                                 GET_SUM_BLOCK(sbi, curseg->segno));
 782         __set_test_and_inuse(sbi, new_segno);
 783
 784         mutex_lock(&dirty_i->seglist_lock);
 785         __remove_dirty_segment(sbi, new_segno, PRE);
 786         __remove_dirty_segment(sbi, new_segno, DIRTY);
 787         mutex_unlock(&dirty_i->seglist_lock);
 788
 789         reset_curseg(sbi, type, 1);
 790         curseg->alloc_type = SSR;
 791         __next_free_blkoff(sbi, curseg, 0);
 792
 793         if (reuse) {
 794                 sum_page = get_sum_page(sbi, new_segno);
 795                 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
 796                 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
 797                 f2fs_put_page(sum_page, 1);
 798         }
 799 }
 800
 801 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
 802 {
 803         struct curseg_info *curseg = CURSEG_I(sbi, type);
 804         const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
 805
 806         if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
 807                 return v_ops->get_victim(sbi,
 808                                 &(curseg)->next_segno, BG_GC, type, SSR);
 809
 810         /* For data segments, let's do SSR more intensively */
 811         for (; type >= CURSEG_HOT_DATA; type--)
 812                 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
 813                                                 BG_GC, type, SSR))
 814                         return 1;
 815         return 0;
 816 }
 817
 818 /*
 819  * flush out current segment and replace it with new segment
 820  * This function should be returned with success, otherwise BUG
 821  */
 822 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
 823                                                 int type, bool force)
 824 {
 825         struct curseg_info *curseg = CURSEG_I(sbi, type);
 826
 827         if (force)
 828                 new_curseg(sbi, type, true);
 829         else if (type == CURSEG_WARM_NODE)
 830                 new_curseg(sbi, type, false);
 831         else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
 832                 new_curseg(sbi, type, false);
 833         else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
 834                 change_curseg(sbi, type, true);
 835         else
 836                 new_curseg(sbi, type, false);
 837
 838         stat_inc_seg_type(sbi, curseg);
 839 }
 840
 841 void allocate_new_segments(struct f2fs_sb_info *sbi)
 842 {
 843         struct curseg_info *curseg;
 844         unsigned int old_curseg;
 845         int i;
 846
 847         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 848                 curseg = CURSEG_I(sbi, i);
 849                 old_curseg = curseg->segno;
 850                 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
 851                 locate_dirty_segment(sbi, old_curseg);
 852         }
 853 }
 854
 855 static const struct segment_allocation default_salloc_ops = {
 856         .allocate_segment = allocate_segment_by_default,
 857 };
 858
 859 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
 860 {
 861         struct curseg_info *curseg = CURSEG_I(sbi, type);
 862         if (curseg->next_blkoff < sbi->blocks_per_seg)
 863                 return true;
 864         return false;
 865 }
 866
 867 static int __get_segment_type_2(struct page *page, enum page_type p_type)
 868 {
 869         if (p_type == DATA)
 870                 return CURSEG_HOT_DATA;
 871         else
 872                 return CURSEG_HOT_NODE;
 873 }
 874
 875 static int __get_segment_type_4(struct page *page, enum page_type p_type)
 876 {
 877         if (p_type == DATA) {
 878                 struct inode *inode = page->mapping->host;
 879
 880                 if (S_ISDIR(inode->i_mode))
 881                         return CURSEG_HOT_DATA;
 882                 else
 883                         return CURSEG_COLD_DATA;
 884         } else {
 885                 if (IS_DNODE(page) && !is_cold_node(page))
 886                         return CURSEG_HOT_NODE;
 887                 else
 888                         return CURSEG_COLD_NODE;
 889         }
 890 }
 891
 892 static int __get_segment_type_6(struct page *page, enum page_type p_type)
 893 {
 894         if (p_type == DATA) {
 895                 struct inode *inode = page->mapping->host;
 896
 897                 if (S_ISDIR(inode->i_mode))
 898                         return CURSEG_HOT_DATA;
 899                 else if (is_cold_data(page) || file_is_cold(inode))
 900                         return CURSEG_COLD_DATA;
 901                 else
 902                         return CURSEG_WARM_DATA;
 903         } else {
 904                 if (IS_DNODE(page))
 905                         return is_cold_node(page) ? CURSEG_WARM_NODE :
 906                                                 CURSEG_HOT_NODE;
 907                 else
 908                         return CURSEG_COLD_NODE;
 909         }
 910 }
 911
 912 static int __get_segment_type(struct page *page, enum page_type p_type)
 913 {
 914         struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
 915         switch (sbi->active_logs) {
 916         case 2:
 917                 return __get_segment_type_2(page, p_type);
 918         case 4:
 919                 return __get_segment_type_4(page, p_type);
 920         }
 921         /* NR_CURSEG_TYPE(6) logs by default */
 922         f2fs_bug_on(sbi->active_logs != NR_CURSEG_TYPE);
 923         return __get_segment_type_6(page, p_type);
 924 }
 925
 926 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
 927                 block_t old_blkaddr, block_t *new_blkaddr,
 928                 struct f2fs_summary *sum, int type)
 929 {
 930         struct sit_info *sit_i = SIT_I(sbi);
 931         struct curseg_info *curseg;
 932         unsigned int old_cursegno;
 933
 934         curseg = CURSEG_I(sbi, type);
 935
 936         mutex_lock(&curseg->curseg_mutex);
 937
 938         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
 939         old_cursegno = curseg->segno;
 940
 941         /*
 942          * __add_sum_entry should be resided under the curseg_mutex
 943          * because, this function updates a summary entry in the
 944          * current summary block.
 945          */
 946         __add_sum_entry(sbi, type, sum);
 947
 948         mutex_lock(&sit_i->sentry_lock);
 949         __refresh_next_blkoff(sbi, curseg);
 950
 951         stat_inc_block_count(sbi, curseg);
 952
 953         if (!__has_curseg_space(sbi, type))
 954                 sit_i->s_ops->allocate_segment(sbi, type, false);
 955         /*
 956          * SIT information should be updated before segment allocation,
 957          * since SSR needs latest valid block information.
 958          */
 959         refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
 960         locate_dirty_segment(sbi, old_cursegno);
 961
 962         mutex_unlock(&sit_i->sentry_lock);
 963
 964         if (page && IS_NODESEG(type))
 965                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
 966
 967         mutex_unlock(&curseg->curseg_mutex);
 968 }
 969
 970 static void do_write_page(struct f2fs_sb_info *sbi, struct page *page,
 971                         block_t old_blkaddr, block_t *new_blkaddr,
 972                         struct f2fs_summary *sum, struct f2fs_io_info *fio)
 973 {
 974         int type = __get_segment_type(page, fio->type);
 975
 976         allocate_data_block(sbi, page, old_blkaddr, new_blkaddr, sum, type);
 977
 978         /* writeout dirty page into bdev */
 979         f2fs_submit_page_mbio(sbi, page, *new_blkaddr, fio);
 980 }
 981
 982 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
 983 {
 984         struct f2fs_io_info fio = {
 985                 .type = META,
 986                 .rw = WRITE_SYNC | REQ_META | REQ_PRIO
 987         };
 988
 989         set_page_writeback(page);
 990         f2fs_submit_page_mbio(sbi, page, page->index, &fio);
 991 }
 992
 993 void write_node_page(struct f2fs_sb_info *sbi, struct page *page,
 994                 struct f2fs_io_info *fio,
 995                 unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr)
 996 {
 997         struct f2fs_summary sum;
 998         set_summary(&sum, nid, 0, 0);
 999         do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, fio);
1000 }
1001
1002 void write_data_page(struct page *page, struct dnode_of_data *dn,
1003                 block_t *new_blkaddr, struct f2fs_io_info *fio)
1004 {
1005         struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
1006         struct f2fs_summary sum;
1007         struct node_info ni;
1008
1009         f2fs_bug_on(dn->data_blkaddr == NULL_ADDR);
1010         get_node_info(sbi, dn->nid, &ni);
1011         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1012
1013         do_write_page(sbi, page, dn->data_blkaddr, new_blkaddr, &sum, fio);
1014 }
1015
1016 void rewrite_data_page(struct page *page, block_t old_blkaddr,
1017                                         struct f2fs_io_info *fio)
1018 {
1019         struct inode *inode = page->mapping->host;
1020         struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
1021         f2fs_submit_page_mbio(sbi, page, old_blkaddr, fio);
1022 }
1023
1024 void recover_data_page(struct f2fs_sb_info *sbi,
1025                         struct page *page, struct f2fs_summary *sum,
1026                         block_t old_blkaddr, block_t new_blkaddr)
1027 {
1028         struct sit_info *sit_i = SIT_I(sbi);
1029         struct curseg_info *curseg;
1030         unsigned int segno, old_cursegno;
1031         struct seg_entry *se;
1032         int type;
1033
1034         segno = GET_SEGNO(sbi, new_blkaddr);
1035         se = get_seg_entry(sbi, segno);
1036         type = se->type;
1037
1038         if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1039                 if (old_blkaddr == NULL_ADDR)
1040                         type = CURSEG_COLD_DATA;
1041                 else
1042                         type = CURSEG_WARM_DATA;
1043         }
1044         curseg = CURSEG_I(sbi, type);
1045
1046         mutex_lock(&curseg->curseg_mutex);
1047         mutex_lock(&sit_i->sentry_lock);
1048
1049         old_cursegno = curseg->segno;
1050
1051         /* change the current segment */
1052         if (segno != curseg->segno) {
1053                 curseg->next_segno = segno;
1054                 change_curseg(sbi, type, true);
1055         }
1056
1057         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1058         __add_sum_entry(sbi, type, sum);
1059
1060         refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
1061         locate_dirty_segment(sbi, old_cursegno);
1062
1063         mutex_unlock(&sit_i->sentry_lock);
1064         mutex_unlock(&curseg->curseg_mutex);
1065 }
1066
1067 void rewrite_node_page(struct f2fs_sb_info *sbi,
1068                         struct page *page, struct f2fs_summary *sum,
1069                         block_t old_blkaddr, block_t new_blkaddr)
1070 {
1071         struct sit_info *sit_i = SIT_I(sbi);
1072         int type = CURSEG_WARM_NODE;
1073         struct curseg_info *curseg;
1074         unsigned int segno, old_cursegno;
1075         block_t next_blkaddr = next_blkaddr_of_node(page);
1076         unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr);
1077         struct f2fs_io_info fio = {
1078                 .type = NODE,
1079                 .rw = WRITE_SYNC,
1080         };
1081
1082         curseg = CURSEG_I(sbi, type);
1083
1084         mutex_lock(&curseg->curseg_mutex);
1085         mutex_lock(&sit_i->sentry_lock);
1086
1087         segno = GET_SEGNO(sbi, new_blkaddr);
1088         old_cursegno = curseg->segno;
1089
1090         /* change the current segment */
1091         if (segno != curseg->segno) {
1092                 curseg->next_segno = segno;
1093                 change_curseg(sbi, type, true);
1094         }
1095         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1096         __add_sum_entry(sbi, type, sum);
1097
1098         /* change the current log to the next block addr in advance */
1099         if (next_segno != segno) {
1100                 curseg->next_segno = next_segno;
1101                 change_curseg(sbi, type, true);
1102         }
1103         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, next_blkaddr);
1104
1105         /* rewrite node page */
1106         set_page_writeback(page);
1107         f2fs_submit_page_mbio(sbi, page, new_blkaddr, &fio);
1108         f2fs_submit_merged_bio(sbi, NODE, WRITE);
1109         refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
1110         locate_dirty_segment(sbi, old_cursegno);
1111
1112         mutex_unlock(&sit_i->sentry_lock);
1113         mutex_unlock(&curseg->curseg_mutex);
1114 }
1115
1116 static inline bool is_merged_page(struct f2fs_sb_info *sbi,
1117                                         struct page *page, enum page_type type)
1118 {
1119         enum page_type btype = PAGE_TYPE_OF_BIO(type);
1120         struct f2fs_bio_info *io = &sbi->write_io[btype];
1121         struct bio_vec *bvec;
1122         int i;
1123
1124         down_read(&io->io_rwsem);
1125         if (!io->bio)
1126                 goto out;
1127
1128         bio_for_each_segment_all(bvec, io->bio, i) {
1129                 if (page == bvec->bv_page) {
1130                         up_read(&io->io_rwsem);
1131                         return true;
1132                 }
1133         }
1134
1135 out:
1136         up_read(&io->io_rwsem);
1137         return false;
1138 }
1139
1140 void f2fs_wait_on_page_writeback(struct page *page,
1141                                 enum page_type type)
1142 {
1143         struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
1144         if (PageWriteback(page)) {
1145                 if (is_merged_page(sbi, page, type))
1146                         f2fs_submit_merged_bio(sbi, type, WRITE);
1147                 wait_on_page_writeback(page);
1148         }
1149 }
1150
1151 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1152 {
1153         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1154         struct curseg_info *seg_i;
1155         unsigned char *kaddr;
1156         struct page *page;
1157         block_t start;
1158         int i, j, offset;
1159
1160         start = start_sum_block(sbi);
1161
1162         page = get_meta_page(sbi, start++);
1163         kaddr = (unsigned char *)page_address(page);
1164
1165         /* Step 1: restore nat cache */
1166         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1167         memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1168
1169         /* Step 2: restore sit cache */
1170         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1171         memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1172                                                 SUM_JOURNAL_SIZE);
1173         offset = 2 * SUM_JOURNAL_SIZE;
1174
1175         /* Step 3: restore summary entries */
1176         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1177                 unsigned short blk_off;
1178                 unsigned int segno;
1179
1180                 seg_i = CURSEG_I(sbi, i);
1181                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1182                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1183                 seg_i->next_segno = segno;
1184                 reset_curseg(sbi, i, 0);
1185                 seg_i->alloc_type = ckpt->alloc_type[i];
1186                 seg_i->next_blkoff = blk_off;
1187
1188                 if (seg_i->alloc_type == SSR)
1189                         blk_off = sbi->blocks_per_seg;
1190
1191                 for (j = 0; j < blk_off; j++) {
1192                         struct f2fs_summary *s;
1193                         s = (struct f2fs_summary *)(kaddr + offset);
1194                         seg_i->sum_blk->entries[j] = *s;
1195                         offset += SUMMARY_SIZE;
1196                         if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1197                                                 SUM_FOOTER_SIZE)
1198                                 continue;
1199
1200                         f2fs_put_page(page, 1);
1201                         page = NULL;
1202
1203                         page = get_meta_page(sbi, start++);
1204                         kaddr = (unsigned char *)page_address(page);
1205                         offset = 0;
1206                 }
1207         }
1208         f2fs_put_page(page, 1);
1209         return 0;
1210 }
1211
1212 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1213 {
1214         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1215         struct f2fs_summary_block *sum;
1216         struct curseg_info *curseg;
1217         struct page *new;
1218         unsigned short blk_off;
1219         unsigned int segno = 0;
1220         block_t blk_addr = 0;
1221
1222         /* get segment number and block addr */
1223         if (IS_DATASEG(type)) {
1224                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1225                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1226                                                         CURSEG_HOT_DATA]);
1227                 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1228                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1229                 else
1230                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1231         } else {
1232                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1233                                                         CURSEG_HOT_NODE]);
1234                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1235                                                         CURSEG_HOT_NODE]);
1236                 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1237                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1238                                                         type - CURSEG_HOT_NODE);
1239                 else
1240                         blk_addr = GET_SUM_BLOCK(sbi, segno);
1241         }
1242
1243         new = get_meta_page(sbi, blk_addr);
1244         sum = (struct f2fs_summary_block *)page_address(new);
1245
1246         if (IS_NODESEG(type)) {
1247                 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) {
1248                         struct f2fs_summary *ns = &sum->entries[0];
1249                         int i;
1250                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1251                                 ns->version = 0;
1252                                 ns->ofs_in_node = 0;
1253                         }
1254                 } else {
1255                         int err;
1256
1257                         err = restore_node_summary(sbi, segno, sum);
1258                         if (err) {
1259                                 f2fs_put_page(new, 1);
1260                                 return err;
1261                         }
1262                 }
1263         }
1264
1265         /* set uncompleted segment to curseg */
1266         curseg = CURSEG_I(sbi, type);
1267         mutex_lock(&curseg->curseg_mutex);
1268         memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1269         curseg->next_segno = segno;
1270         reset_curseg(sbi, type, 0);
1271         curseg->alloc_type = ckpt->alloc_type[type];
1272         curseg->next_blkoff = blk_off;
1273         mutex_unlock(&curseg->curseg_mutex);
1274         f2fs_put_page(new, 1);
1275         return 0;
1276 }
1277
1278 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1279 {
1280         int type = CURSEG_HOT_DATA;
1281         int err;
1282
1283         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1284                 /* restore for compacted data summary */
1285                 if (read_compacted_summaries(sbi))
1286                         return -EINVAL;
1287                 type = CURSEG_HOT_NODE;
1288         }
1289
1290         for (; type <= CURSEG_COLD_NODE; type++) {
1291                 err = read_normal_summaries(sbi, type);
1292                 if (err)
1293                         return err;
1294         }
1295
1296         return 0;
1297 }
1298
1299 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1300 {
1301         struct page *page;
1302         unsigned char *kaddr;
1303         struct f2fs_summary *summary;
1304         struct curseg_info *seg_i;
1305         int written_size = 0;
1306         int i, j;
1307
1308         page = grab_meta_page(sbi, blkaddr++);
1309         kaddr = (unsigned char *)page_address(page);
1310
1311         /* Step 1: write nat cache */
1312         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1313         memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1314         written_size += SUM_JOURNAL_SIZE;
1315
1316         /* Step 2: write sit cache */
1317         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1318         memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1319                                                 SUM_JOURNAL_SIZE);
1320         written_size += SUM_JOURNAL_SIZE;
1321
1322         /* Step 3: write summary entries */
1323         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1324                 unsigned short blkoff;
1325                 seg_i = CURSEG_I(sbi, i);
1326                 if (sbi->ckpt->alloc_type[i] == SSR)
1327                         blkoff = sbi->blocks_per_seg;
1328                 else
1329                         blkoff = curseg_blkoff(sbi, i);
1330
1331                 for (j = 0; j < blkoff; j++) {
1332                         if (!page) {
1333                                 page = grab_meta_page(sbi, blkaddr++);
1334                                 kaddr = (unsigned char *)page_address(page);
1335                                 written_size = 0;
1336                         }
1337                         summary = (struct f2fs_summary *)(kaddr + written_size);
1338                         *summary = seg_i->sum_blk->entries[j];
1339                         written_size += SUMMARY_SIZE;
1340
1341                         if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1342                                                         SUM_FOOTER_SIZE)
1343                                 continue;
1344
1345                         set_page_dirty(page);
1346                         f2fs_put_page(page, 1);
1347                         page = NULL;
1348                 }
1349         }
1350         if (page) {
1351                 set_page_dirty(page);
1352                 f2fs_put_page(page, 1);
1353         }
1354 }
1355
1356 static void write_normal_summaries(struct f2fs_sb_info *sbi,
1357                                         block_t blkaddr, int type)
1358 {
1359         int i, end;
1360         if (IS_DATASEG(type))
1361                 end = type + NR_CURSEG_DATA_TYPE;
1362         else
1363                 end = type + NR_CURSEG_NODE_TYPE;
1364
1365         for (i = type; i < end; i++) {
1366                 struct curseg_info *sum = CURSEG_I(sbi, i);
1367                 mutex_lock(&sum->curseg_mutex);
1368                 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1369                 mutex_unlock(&sum->curseg_mutex);
1370         }
1371 }
1372
1373 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1374 {
1375         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1376                 write_compacted_summaries(sbi, start_blk);
1377         else
1378                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1379 }
1380
1381 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1382 {
1383         if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG))
1384                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1385 }
1386
1387 int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1388                                         unsigned int val, int alloc)
1389 {
1390         int i;
1391
1392         if (type == NAT_JOURNAL) {
1393                 for (i = 0; i < nats_in_cursum(sum); i++) {
1394                         if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1395                                 return i;
1396                 }
1397                 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)
1398                         return update_nats_in_cursum(sum, 1);
1399         } else if (type == SIT_JOURNAL) {
1400                 for (i = 0; i < sits_in_cursum(sum); i++)
1401                         if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1402                                 return i;
1403                 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)
1404                         return update_sits_in_cursum(sum, 1);
1405         }
1406         return -1;
1407 }
1408
1409 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1410                                         unsigned int segno)
1411 {
1412         struct sit_info *sit_i = SIT_I(sbi);
1413         unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno);
1414         block_t blk_addr = sit_i->sit_base_addr + offset;
1415
1416         check_seg_range(sbi, segno);
1417
1418         /* calculate sit block address */
1419         if (f2fs_test_bit(offset, sit_i->sit_bitmap))
1420                 blk_addr += sit_i->sit_blocks;
1421
1422         return get_meta_page(sbi, blk_addr);
1423 }
1424
1425 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1426                                         unsigned int start)
1427 {
1428         struct sit_info *sit_i = SIT_I(sbi);
1429         struct page *src_page, *dst_page;
1430         pgoff_t src_off, dst_off;
1431         void *src_addr, *dst_addr;
1432
1433         src_off = current_sit_addr(sbi, start);
1434         dst_off = next_sit_addr(sbi, src_off);
1435
1436         /* get current sit block page without lock */
1437         src_page = get_meta_page(sbi, src_off);
1438         dst_page = grab_meta_page(sbi, dst_off);
1439         f2fs_bug_on(PageDirty(src_page));
1440
1441         src_addr = page_address(src_page);
1442         dst_addr = page_address(dst_page);
1443         memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1444
1445         set_page_dirty(dst_page);
1446         f2fs_put_page(src_page, 1);
1447
1448         set_to_next_sit(sit_i, start);
1449
1450         return dst_page;
1451 }
1452
1453 static bool flush_sits_in_journal(struct f2fs_sb_info *sbi)
1454 {
1455         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1456         struct f2fs_summary_block *sum = curseg->sum_blk;
1457         int i;
1458
1459         /*
1460          * If the journal area in the current summary is full of sit entries,
1461          * all the sit entries will be flushed. Otherwise the sit entries
1462          * are not able to replace with newly hot sit entries.
1463          */
1464         if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) {
1465                 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1466                         unsigned int segno;
1467                         segno = le32_to_cpu(segno_in_journal(sum, i));
1468                         __mark_sit_entry_dirty(sbi, segno);
1469                 }
1470                 update_sits_in_cursum(sum, -sits_in_cursum(sum));
1471                 return true;
1472         }
1473         return false;
1474 }
1475
1476 /*
1477  * CP calls this function, which flushes SIT entries including sit_journal,
1478  * and moves prefree segs to free segs.
1479  */
1480 void flush_sit_entries(struct f2fs_sb_info *sbi)
1481 {
1482         struct sit_info *sit_i = SIT_I(sbi);
1483         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1484         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1485         struct f2fs_summary_block *sum = curseg->sum_blk;
1486         unsigned long nsegs = TOTAL_SEGS(sbi);
1487         struct page *page = NULL;
1488         struct f2fs_sit_block *raw_sit = NULL;
1489         unsigned int start = 0, end = 0;
1490         unsigned int segno = -1;
1491         bool flushed;
1492
1493         mutex_lock(&curseg->curseg_mutex);
1494         mutex_lock(&sit_i->sentry_lock);
1495
1496         /*
1497          * "flushed" indicates whether sit entries in journal are flushed
1498          * to the SIT area or not.
1499          */
1500         flushed = flush_sits_in_journal(sbi);
1501
1502         while ((segno = find_next_bit(bitmap, nsegs, segno + 1)) < nsegs) {
1503                 struct seg_entry *se = get_seg_entry(sbi, segno);
1504                 int sit_offset, offset;
1505
1506                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1507
1508                 /* add discard candidates */
1509                 if (SM_I(sbi)->nr_discards < SM_I(sbi)->max_discards)
1510                         add_discard_addrs(sbi, segno, se);
1511
1512                 if (flushed)
1513                         goto to_sit_page;
1514
1515                 offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1);
1516                 if (offset >= 0) {
1517                         segno_in_journal(sum, offset) = cpu_to_le32(segno);
1518                         seg_info_to_raw_sit(se, &sit_in_journal(sum, offset));
1519                         goto flush_done;
1520                 }
1521 to_sit_page:
1522                 if (!page || (start > segno) || (segno > end)) {
1523                         if (page) {
1524                                 f2fs_put_page(page, 1);
1525                                 page = NULL;
1526                         }
1527
1528                         start = START_SEGNO(sit_i, segno);
1529                         end = start + SIT_ENTRY_PER_BLOCK - 1;
1530
1531                         /* read sit block that will be updated */
1532                         page = get_next_sit_page(sbi, start);
1533                         raw_sit = page_address(page);
1534                 }
1535
1536                 /* udpate entry in SIT block */
1537                 seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]);
1538 flush_done:
1539                 __clear_bit(segno, bitmap);
1540                 sit_i->dirty_sentries--;
1541         }
1542         mutex_unlock(&sit_i->sentry_lock);
1543         mutex_unlock(&curseg->curseg_mutex);
1544
1545         /* writeout last modified SIT block */
1546         f2fs_put_page(page, 1);
1547
1548         set_prefree_as_free_segments(sbi);
1549 }
1550
1551 static int build_sit_info(struct f2fs_sb_info *sbi)
1552 {
1553         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1554         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1555         struct sit_info *sit_i;
1556         unsigned int sit_segs, start;
1557         char *src_bitmap, *dst_bitmap;
1558         unsigned int bitmap_size;
1559
1560         /* allocate memory for SIT information */
1561         sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
1562         if (!sit_i)
1563                 return -ENOMEM;
1564
1565         SM_I(sbi)->sit_info = sit_i;
1566
1567         sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry));
1568         if (!sit_i->sentries)
1569                 return -ENOMEM;
1570
1571         bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1572         sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1573         if (!sit_i->dirty_sentries_bitmap)
1574                 return -ENOMEM;
1575
1576         for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1577                 sit_i->sentries[start].cur_valid_map
1578                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1579                 sit_i->sentries[start].ckpt_valid_map
1580                         = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1581                 if (!sit_i->sentries[start].cur_valid_map
1582                                 || !sit_i->sentries[start].ckpt_valid_map)
1583                         return -ENOMEM;
1584         }
1585
1586         if (sbi->segs_per_sec > 1) {
1587                 sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) *
1588                                         sizeof(struct sec_entry));
1589                 if (!sit_i->sec_entries)
1590                         return -ENOMEM;
1591         }
1592
1593         /* get information related with SIT */
1594         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
1595
1596         /* setup SIT bitmap from ckeckpoint pack */
1597         bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
1598         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
1599
1600         dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
1601         if (!dst_bitmap)
1602                 return -ENOMEM;
1603
1604         /* init SIT information */
1605         sit_i->s_ops = &default_salloc_ops;
1606
1607         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
1608         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
1609         sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
1610         sit_i->sit_bitmap = dst_bitmap;
1611         sit_i->bitmap_size = bitmap_size;
1612         sit_i->dirty_sentries = 0;
1613         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
1614         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
1615         sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
1616         mutex_init(&sit_i->sentry_lock);
1617         return 0;
1618 }
1619
1620 static int build_free_segmap(struct f2fs_sb_info *sbi)
1621 {
1622         struct f2fs_sm_info *sm_info = SM_I(sbi);
1623         struct free_segmap_info *free_i;
1624         unsigned int bitmap_size, sec_bitmap_size;
1625
1626         /* allocate memory for free segmap information */
1627         free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
1628         if (!free_i)
1629                 return -ENOMEM;
1630
1631         SM_I(sbi)->free_info = free_i;
1632
1633         bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1634         free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL);
1635         if (!free_i->free_segmap)
1636                 return -ENOMEM;
1637
1638         sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1639         free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL);
1640         if (!free_i->free_secmap)
1641                 return -ENOMEM;
1642
1643         /* set all segments as dirty temporarily */
1644         memset(free_i->free_segmap, 0xff, bitmap_size);
1645         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
1646
1647         /* init free segmap information */
1648         free_i->start_segno =
1649                 (unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr);
1650         free_i->free_segments = 0;
1651         free_i->free_sections = 0;
1652         rwlock_init(&free_i->segmap_lock);
1653         return 0;
1654 }
1655
1656 static int build_curseg(struct f2fs_sb_info *sbi)
1657 {
1658         struct curseg_info *array;
1659         int i;
1660
1661         array = kzalloc(sizeof(*array) * NR_CURSEG_TYPE, GFP_KERNEL);
1662         if (!array)
1663                 return -ENOMEM;
1664
1665         SM_I(sbi)->curseg_array = array;
1666
1667         for (i = 0; i < NR_CURSEG_TYPE; i++) {
1668                 mutex_init(&array[i].curseg_mutex);
1669                 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
1670                 if (!array[i].sum_blk)
1671                         return -ENOMEM;
1672                 array[i].segno = NULL_SEGNO;
1673                 array[i].next_blkoff = 0;
1674         }
1675         return restore_curseg_summaries(sbi);
1676 }
1677
1678 static void build_sit_entries(struct f2fs_sb_info *sbi)
1679 {
1680         struct sit_info *sit_i = SIT_I(sbi);
1681         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1682         struct f2fs_summary_block *sum = curseg->sum_blk;
1683         int sit_blk_cnt = SIT_BLK_CNT(sbi);
1684         unsigned int i, start, end;
1685         unsigned int readed, start_blk = 0;
1686         int nrpages = MAX_BIO_BLOCKS(max_hw_blocks(sbi));
1687
1688         do {
1689                 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT);
1690
1691                 start = start_blk * sit_i->sents_per_block;
1692                 end = (start_blk + readed) * sit_i->sents_per_block;
1693
1694                 for (; start < end && start < TOTAL_SEGS(sbi); start++) {
1695                         struct seg_entry *se = &sit_i->sentries[start];
1696                         struct f2fs_sit_block *sit_blk;
1697                         struct f2fs_sit_entry sit;
1698                         struct page *page;
1699
1700                         mutex_lock(&curseg->curseg_mutex);
1701                         for (i = 0; i < sits_in_cursum(sum); i++) {
1702                                 if (le32_to_cpu(segno_in_journal(sum, i))
1703                                                                 == start) {
1704                                         sit = sit_in_journal(sum, i);
1705                                         mutex_unlock(&curseg->curseg_mutex);
1706                                         goto got_it;
1707                                 }
1708                         }
1709                         mutex_unlock(&curseg->curseg_mutex);
1710
1711                         page = get_current_sit_page(sbi, start);
1712                         sit_blk = (struct f2fs_sit_block *)page_address(page);
1713                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
1714                         f2fs_put_page(page, 1);
1715 got_it:
1716                         check_block_count(sbi, start, &sit);
1717                         seg_info_from_raw_sit(se, &sit);
1718                         if (sbi->segs_per_sec > 1) {
1719                                 struct sec_entry *e = get_sec_entry(sbi, start);
1720                                 e->valid_blocks += se->valid_blocks;
1721                         }
1722                 }
1723                 start_blk += readed;
1724         } while (start_blk < sit_blk_cnt);
1725 }
1726
1727 static void init_free_segmap(struct f2fs_sb_info *sbi)
1728 {
1729         unsigned int start;
1730         int type;
1731
1732         for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1733                 struct seg_entry *sentry = get_seg_entry(sbi, start);
1734                 if (!sentry->valid_blocks)
1735                         __set_free(sbi, start);
1736         }
1737
1738         /* set use the current segments */
1739         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
1740                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
1741                 __set_test_and_inuse(sbi, curseg_t->segno);
1742         }
1743 }
1744
1745 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
1746 {
1747         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1748         struct free_segmap_info *free_i = FREE_I(sbi);
1749         unsigned int segno = 0, offset = 0, total_segs = TOTAL_SEGS(sbi);
1750         unsigned short valid_blocks;
1751
1752         while (1) {
1753                 /* find dirty segment based on free segmap */
1754                 segno = find_next_inuse(free_i, total_segs, offset);
1755                 if (segno >= total_segs)
1756                         break;
1757                 offset = segno + 1;
1758                 valid_blocks = get_valid_blocks(sbi, segno, 0);
1759                 if (valid_blocks >= sbi->blocks_per_seg || !valid_blocks)
1760                         continue;
1761                 mutex_lock(&dirty_i->seglist_lock);
1762                 __locate_dirty_segment(sbi, segno, DIRTY);
1763                 mutex_unlock(&dirty_i->seglist_lock);
1764         }
1765 }
1766
1767 static int init_victim_secmap(struct f2fs_sb_info *sbi)
1768 {
1769         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1770         unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1771
1772         dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL);
1773         if (!dirty_i->victim_secmap)
1774                 return -ENOMEM;
1775         return 0;
1776 }
1777
1778 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
1779 {
1780         struct dirty_seglist_info *dirty_i;
1781         unsigned int bitmap_size, i;
1782
1783         /* allocate memory for dirty segments list information */
1784         dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
1785         if (!dirty_i)
1786                 return -ENOMEM;
1787
1788         SM_I(sbi)->dirty_info = dirty_i;
1789         mutex_init(&dirty_i->seglist_lock);
1790
1791         bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1792
1793         for (i = 0; i < NR_DIRTY_TYPE; i++) {
1794                 dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL);
1795                 if (!dirty_i->dirty_segmap[i])
1796                         return -ENOMEM;
1797         }
1798
1799         init_dirty_segmap(sbi);
1800         return init_victim_secmap(sbi);
1801 }
1802
1803 /*
1804  * Update min, max modified time for cost-benefit GC algorithm
1805  */
1806 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
1807 {
1808         struct sit_info *sit_i = SIT_I(sbi);
1809         unsigned int segno;
1810
1811         mutex_lock(&sit_i->sentry_lock);
1812
1813         sit_i->min_mtime = LLONG_MAX;
1814
1815         for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) {
1816                 unsigned int i;
1817                 unsigned long long mtime = 0;
1818
1819                 for (i = 0; i < sbi->segs_per_sec; i++)
1820                         mtime += get_seg_entry(sbi, segno + i)->mtime;
1821
1822                 mtime = div_u64(mtime, sbi->segs_per_sec);
1823
1824                 if (sit_i->min_mtime > mtime)
1825                         sit_i->min_mtime = mtime;
1826         }
1827         sit_i->max_mtime = get_mtime(sbi);
1828         mutex_unlock(&sit_i->sentry_lock);
1829 }
1830
1831 int build_segment_manager(struct f2fs_sb_info *sbi)
1832 {
1833         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1834         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1835         dev_t dev = sbi->sb->s_bdev->bd_dev;
1836         struct f2fs_sm_info *sm_info;
1837         int err;
1838
1839         sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
1840         if (!sm_info)
1841                 return -ENOMEM;
1842
1843         /* init sm info */
1844         sbi->sm_info = sm_info;
1845         INIT_LIST_HEAD(&sm_info->wblist_head);
1846         spin_lock_init(&sm_info->wblist_lock);
1847         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
1848         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
1849         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
1850         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
1851         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
1852         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
1853         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
1854         sm_info->rec_prefree_segments = sm_info->main_segments *
1855                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
1856         sm_info->ipu_policy = F2FS_IPU_DISABLE;
1857         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
1858
1859         INIT_LIST_HEAD(&sm_info->discard_list);
1860         sm_info->nr_discards = 0;
1861         sm_info->max_discards = 0;
1862
1863         if (test_opt(sbi, FLUSH_MERGE)) {
1864                 spin_lock_init(&sm_info->issue_lock);
1865                 init_waitqueue_head(&sm_info->flush_wait_queue);
1866
1867                 sm_info->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
1868                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
1869                 if (IS_ERR(sm_info->f2fs_issue_flush))
1870                         return PTR_ERR(sm_info->f2fs_issue_flush);
1871         }
1872
1873         err = build_sit_info(sbi);
1874         if (err)
1875                 return err;
1876         err = build_free_segmap(sbi);
1877         if (err)
1878                 return err;
1879         err = build_curseg(sbi);
1880         if (err)
1881                 return err;
1882
1883         /* reinit free segmap based on SIT */
1884         build_sit_entries(sbi);
1885
1886         init_free_segmap(sbi);
1887         err = build_dirty_segmap(sbi);
1888         if (err)
1889                 return err;
1890
1891         init_min_max_mtime(sbi);
1892         return 0;
1893 }
1894
1895 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
1896                 enum dirty_type dirty_type)
1897 {
1898         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1899
1900         mutex_lock(&dirty_i->seglist_lock);
1901         kfree(dirty_i->dirty_segmap[dirty_type]);
1902         dirty_i->nr_dirty[dirty_type] = 0;
1903         mutex_unlock(&dirty_i->seglist_lock);
1904 }
1905
1906 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
1907 {
1908         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1909         kfree(dirty_i->victim_secmap);
1910 }
1911
1912 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
1913 {
1914         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1915         int i;
1916
1917         if (!dirty_i)
1918                 return;
1919
1920         /* discard pre-free/dirty segments list */
1921         for (i = 0; i < NR_DIRTY_TYPE; i++)
1922                 discard_dirty_segmap(sbi, i);
1923
1924         destroy_victim_secmap(sbi);
1925         SM_I(sbi)->dirty_info = NULL;
1926         kfree(dirty_i);
1927 }
1928
1929 static void destroy_curseg(struct f2fs_sb_info *sbi)
1930 {
1931         struct curseg_info *array = SM_I(sbi)->curseg_array;
1932         int i;
1933
1934         if (!array)
1935                 return;
1936         SM_I(sbi)->curseg_array = NULL;
1937         for (i = 0; i < NR_CURSEG_TYPE; i++)
1938                 kfree(array[i].sum_blk);
1939         kfree(array);
1940 }
1941
1942 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
1943 {
1944         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
1945         if (!free_i)
1946                 return;
1947         SM_I(sbi)->free_info = NULL;
1948         kfree(free_i->free_segmap);
1949         kfree(free_i->free_secmap);
1950         kfree(free_i);
1951 }
1952
1953 static void destroy_sit_info(struct f2fs_sb_info *sbi)
1954 {
1955         struct sit_info *sit_i = SIT_I(sbi);
1956         unsigned int start;
1957
1958         if (!sit_i)
1959                 return;
1960
1961         if (sit_i->sentries) {
1962                 for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1963                         kfree(sit_i->sentries[start].cur_valid_map);
1964                         kfree(sit_i->sentries[start].ckpt_valid_map);
1965                 }
1966         }
1967         vfree(sit_i->sentries);
1968         vfree(sit_i->sec_entries);
1969         kfree(sit_i->dirty_sentries_bitmap);
1970
1971         SM_I(sbi)->sit_info = NULL;
1972         kfree(sit_i->sit_bitmap);
1973         kfree(sit_i);
1974 }
1975
1976 void destroy_segment_manager(struct f2fs_sb_info *sbi)
1977 {
1978         struct f2fs_sm_info *sm_info = SM_I(sbi);
1979         if (!sm_info)
1980                 return;
1981         if (sm_info->f2fs_issue_flush)
1982                 kthread_stop(sm_info->f2fs_issue_flush);
1983         destroy_dirty_segmap(sbi);
1984         destroy_curseg(sbi);
1985         destroy_free_segmap(sbi);
1986         destroy_sit_info(sbi);
1987         sbi->sm_info = NULL;
1988         kfree(sm_info);
1989 }
1990
1991 int __init create_segment_manager_caches(void)
1992 {
1993         discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
1994                         sizeof(struct discard_entry));
1995         if (!discard_entry_slab)
1996                 return -ENOMEM;
1997         flush_cmd_slab = f2fs_kmem_cache_create("flush_command",
1998                         sizeof(struct flush_cmd));
1999         if (!flush_cmd_slab) {
2000                 kmem_cache_destroy(discard_entry_slab);
2001                 return -ENOMEM;
2002         }
2003         return 0;
2004 }
2005
2006 void destroy_segment_manager_caches(void)
2007 {
2008         kmem_cache_destroy(discard_entry_slab);
2009         kmem_cache_destroy(flush_cmd_slab);
2010 }