[PATCH] mm: microopt conditions

[cascardo/linux.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3b21a13..b0647b5 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -336,9 +336,9 @@ static inline void __free_pages_bulk (struct page *page,
 
 static inline int free_pages_check(const char *function, struct page *page)
 {
-       if (    page_mapcount(page) ||
-               page->mapping != NULL ||
-               page_count(page) != 0 ||
+       if (unlikely(page_mapcount(page) |
+               (page->mapping != NULL)  |
+               (page_count(page) != 0)  |
                (page->flags & (
                        1 << PG_lru     |
                        1 << PG_private |
@@ -348,7 +348,7 @@ static inline int free_pages_check(const char *function, struct page *page)
                        1 << PG_slab    |
                        1 << PG_swapcache |
                        1 << PG_writeback |
-                       1 << PG_reserved )))
+                       1 << PG_reserved ))))
                bad_page(function, page);
        if (PageDirty(page))
                __ClearPageDirty(page);
@@ -375,11 +375,10 @@ static int
 free_pages_bulk(struct zone *zone, int count,
                struct list_head *list, unsigned int order)
 {
-       unsigned long flags;
        struct page *page = NULL;
        int ret = 0;
 
-       spin_lock_irqsave(&zone->lock, flags);
+       spin_lock(&zone->lock);
        zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
        while (!list_empty(list) && count--) {
@@ -389,12 +388,13 @@ free_pages_bulk(struct zone *zone, int count,
                __free_pages_bulk(page, zone, order);
                ret++;
        }
-       spin_unlock_irqrestore(&zone->lock, flags);
+       spin_unlock(&zone->lock);
        return ret;
 }
 
 void __free_pages_ok(struct page *page, unsigned int order)
 {
+       unsigned long flags;
        LIST_HEAD(list);
        int i;
        int reserved = 0;
@@ -415,7 +415,9 @@ void __free_pages_ok(struct page *page, unsigned int order)
        list_add(&page->lru, &list);
        mod_page_state(pgfree, 1 << order);
        kernel_map_pages(page, 1<<order, 0);
+       local_irq_save(flags);
        free_pages_bulk(page_zone(page), 1, &list, order);
+       local_irq_restore(flags);
 }
 
 
@@ -451,31 +453,14 @@ expand(struct zone *zone, struct page *page,
        return page;
 }
 
-void set_page_refs(struct page *page, int order)
-{
-#ifdef CONFIG_MMU
-       set_page_count(page, 1);
-#else
-       int i;
-
-       /*
-        * We need to reference all the pages for this order, otherwise if
-        * anyone accesses one of the pages with (get/put) it will be freed.
-        * - eg: access_process_vm()
-        */
-       for (i = 0; i < (1 << order); i++)
-               set_page_count(page + i, 1);
-#endif /* CONFIG_MMU */
-}
-
 /*
  * This page is about to be returned from the page allocator
  */
 static int prep_new_page(struct page *page, int order)
 {
-       if (    page_mapcount(page) ||
-               page->mapping != NULL ||
-               page_count(page) != 0 ||
+       if (unlikely(page_mapcount(page) |
+               (page->mapping != NULL)  |
+               (page_count(page) != 0)  |
                (page->flags & (
                        1 << PG_lru     |
                        1 << PG_private |
@@ -486,7 +471,7 @@ static int prep_new_page(struct page *page, int order)
                        1 << PG_slab    |
                        1 << PG_swapcache |
                        1 << PG_writeback |
-                       1 << PG_reserved )))
+                       1 << PG_reserved ))))
                bad_page(__FUNCTION__, page);
 
        /*
@@ -539,12 +524,11 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order)
 static int rmqueue_bulk(struct zone *zone, unsigned int order, 
                        unsigned long count, struct list_head *list)
 {
-       unsigned long flags;
        int i;
        int allocated = 0;
        struct page *page;
        
-       spin_lock_irqsave(&zone->lock, flags);
+       spin_lock(&zone->lock);
        for (i = 0; i < count; ++i) {
                page = __rmqueue(zone, order);
                if (page == NULL)
@@ -552,7 +536,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                allocated++;
                list_add_tail(&page->lru, list);
        }
-       spin_unlock_irqrestore(&zone->lock, flags);
+       spin_unlock(&zone->lock);
        return allocated;
 }
 
@@ -589,6 +573,7 @@ void drain_remote_pages(void)
 #if defined(CONFIG_PM) || defined(CONFIG_HOTPLUG_CPU)
 static void __drain_pages(unsigned int cpu)
 {
+       unsigned long flags;
        struct zone *zone;
        int i;
 
@@ -600,8 +585,10 @@ static void __drain_pages(unsigned int cpu)
                        struct per_cpu_pages *pcp;
 
                        pcp = &pset->pcp[i];
+                       local_irq_save(flags);
                        pcp->count -= free_pages_bulk(zone, pcp->count,
                                                &pcp->list, 0);
+                       local_irq_restore(flags);
                }
        }
 }
@@ -744,7 +731,7 @@ again:
                if (pcp->count <= pcp->low)
                        pcp->count += rmqueue_bulk(zone, 0,
                                                pcp->batch, &pcp->list);
-               if (pcp->count) {
+               if (likely(pcp->count)) {
                        page = list_entry(pcp->list.next, struct page, lru);
                        list_del(&page->lru);
                        pcp->count--;
@@ -903,8 +890,7 @@ restart:
                alloc_flags |= ALLOC_HARDER;
        if (gfp_mask & __GFP_HIGH)
                alloc_flags |= ALLOC_HIGH;
-       if (wait)
-               alloc_flags |= ALLOC_CPUSET;
+       alloc_flags |= ALLOC_CPUSET;
 
        /*
         * Go through the zonelist again. Let __GFP_HIGH and allocations
@@ -926,7 +912,7 @@ restart:
 nofail_alloc:
                        /* go through the zonelist yet again, ignoring mins */
                        page = get_page_from_freelist(gfp_mask, order,
-                               zonelist, ALLOC_NO_WATERMARKS|ALLOC_CPUSET);
+                               zonelist, ALLOC_NO_WATERMARKS);
                        if (page)
                                goto got_pg;
                        if (gfp_mask & __GFP_NOFAIL) {
@@ -1709,8 +1695,6 @@ void __devinit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
        for (pfn = start_pfn; pfn < end_pfn; pfn++, page++) {
                if (!early_pfn_valid(pfn))
                        continue;
-               if (!early_pfn_in_nid(pfn, nid))
-                       continue;
                page = pfn_to_page(pfn);
                set_page_links(page, zone, nid, pfn);
                set_page_count(page, 1);
@@ -1896,7 +1880,7 @@ static int __devinit pageset_cpuup_callback(struct notifier_block *nfb,
 static struct notifier_block pageset_notifier =
        { &pageset_cpuup_callback, NULL, 0 };
 
-void __init setup_per_cpu_pageset()
+void __init setup_per_cpu_pageset(void)
 {
        int err;