[PATCH] cfq-iosched: kill the empty_list

[cascardo/linux.git] / block / cfq-iosched.c

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 85f1d87..6fb1613 100644 (file)
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -81,11 +81,6 @@ struct cfq_data {
        struct list_head idle_rr;
        unsigned int busy_queues;
 
-       /*
-        * non-ordered list of empty cfqq's
-        */
-       struct list_head empty_list;
-
        /*
         * cfqq lookup hash
         */
@@ -136,7 +131,7 @@ struct cfq_queue {
        struct hlist_node cfq_hash;
        /* hash key */
        unsigned int key;
-       /* on either rr or empty list of cfqd */
+       /* member of the rr/busy/cur/idle cfqd list */
        struct list_head cfq_list;
        /* sorted list of pending requests */
        struct rb_root sort_list;
@@ -152,7 +147,6 @@ struct cfq_queue {
        unsigned long slice_start;
        unsigned long slice_end;
        unsigned long slice_left;
-       unsigned long service_last;
 
        /* number of requests that are on the dispatch list */
        int on_dispatch[2];
@@ -174,6 +168,7 @@ enum cfqq_state_flags {
        CFQ_CFQQ_FLAG_fifo_expire,
        CFQ_CFQQ_FLAG_idle_window,
        CFQ_CFQQ_FLAG_prio_changed,
+       CFQ_CFQQ_FLAG_queue_new,
 };
 
 #define CFQ_CFQQ_FNS(name)                                             \
@@ -198,6 +193,7 @@ CFQ_CFQQ_FNS(must_dispatch);
 CFQ_CFQQ_FNS(fifo_expire);
 CFQ_CFQQ_FNS(idle_window);
 CFQ_CFQQ_FNS(prio_changed);
+CFQ_CFQQ_FNS(queue_new);
 #undef CFQ_CFQQ_FNS
 
 static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short);
@@ -350,7 +346,7 @@ cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 static void cfq_resort_rr_list(struct cfq_queue *cfqq, int preempted)
 {
        struct cfq_data *cfqd = cfqq->cfqd;
-       struct list_head *list, *entry;
+       struct list_head *list;
 
        BUG_ON(!cfq_cfqq_on_rr(cfqq));
 
@@ -375,31 +371,26 @@ static void cfq_resort_rr_list(struct cfq_queue *cfqq, int preempted)
        }
 
        /*
-        * if queue was preempted, just add to front to be fair. busy_rr
-        * isn't sorted, but insert at the back for fairness.
+        * If this queue was preempted or is new (never been serviced), let
+        * it be added first for fairness but beind other new queues.
+        * Otherwise, just add to the back  of the list.
         */
-       if (preempted || list == &cfqd->busy_rr) {
-               if (preempted)
-                       list = list->prev;
+       if (preempted || cfq_cfqq_queue_new(cfqq)) {
+               struct list_head *n = list;
+               struct cfq_queue *__cfqq;
 
-               list_add_tail(&cfqq->cfq_list, list);
-               return;
-       }
+               while (n->next != list) {
+                       __cfqq = list_entry_cfqq(n->next);
+                       if (!cfq_cfqq_queue_new(__cfqq))
+                               break;
 
-       /*
-        * sort by when queue was last serviced
-        */
-       entry = list;
-       while ((entry = entry->prev) != list) {
-               struct cfq_queue *__cfqq = list_entry_cfqq(entry);
+                       n = n->next;
+               }
 
-               if (!__cfqq->service_last)
-                       break;
-               if (time_before(__cfqq->service_last, cfqq->service_last))
-                       break;
+               list = n;
        }
 
-       list_add(&cfqq->cfq_list, entry);
+       list_add_tail(&cfqq->cfq_list, list);
 }
 
 /*
@@ -421,7 +412,7 @@ cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
        BUG_ON(!cfq_cfqq_on_rr(cfqq));
        cfq_clear_cfqq_on_rr(cfqq);
-       list_move(&cfqq->cfq_list, &cfqd->empty_list);
+       list_del_init(&cfqq->cfq_list);
 
        BUG_ON(!cfqd->busy_queues);
        cfqd->busy_queues--;
@@ -591,13 +582,12 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
        if (cfq_cfqq_wait_request(cfqq))
                del_timer(&cfqd->idle_slice_timer);
 
-       if (!preempted && !cfq_cfqq_dispatched(cfqq)) {
-               cfqq->service_last = now;
+       if (!preempted && !cfq_cfqq_dispatched(cfqq))
                cfq_schedule_dispatch(cfqd);
-       }
 
        cfq_clear_cfqq_must_dispatch(cfqq);
        cfq_clear_cfqq_wait_request(cfqq);
+       cfq_clear_cfqq_queue_new(cfqq);
 
        /*
         * store what was left of this slice, if the queue idled out
@@ -1148,8 +1138,9 @@ static void cfq_exit_io_context(struct io_context *ioc)
 static struct cfq_io_context *
 cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
 {
-       struct cfq_io_context *cic = kmem_cache_alloc(cfq_ioc_pool, gfp_mask);
+       struct cfq_io_context *cic;
 
+       cic = kmem_cache_alloc_node(cfq_ioc_pool, gfp_mask, cfqd->queue->node);
        if (cic) {
                memset(cic, 0, sizeof(*cic));
                cic->last_end_request = jiffies;
@@ -1277,11 +1268,11 @@ retry:
                         * free memory.
                         */
                        spin_unlock_irq(cfqd->queue->queue_lock);
-                       new_cfqq = kmem_cache_alloc(cfq_pool, gfp_mask|__GFP_NOFAIL);
+                       new_cfqq = kmem_cache_alloc_node(cfq_pool, gfp_mask|__GFP_NOFAIL, cfqd->queue->node);
                        spin_lock_irq(cfqd->queue->queue_lock);
                        goto retry;
                } else {
-                       cfqq = kmem_cache_alloc(cfq_pool, gfp_mask);
+                       cfqq = kmem_cache_alloc_node(cfq_pool, gfp_mask, cfqd->queue->node);
                        if (!cfqq)
                                goto out;
                }
@@ -1296,13 +1287,13 @@ retry:
                hlist_add_head(&cfqq->cfq_hash, &cfqd->cfq_hash[hashval]);
                atomic_set(&cfqq->ref, 0);
                cfqq->cfqd = cfqd;
-               cfqq->service_last = 0;
                /*
                 * set ->slice_left to allow preemption for a new process
                 */
                cfqq->slice_left = 2 * cfqd->cfq_slice_idle;
                cfq_mark_cfqq_idle_window(cfqq);
                cfq_mark_cfqq_prio_changed(cfqq);
+               cfq_mark_cfqq_queue_new(cfqq);
                cfq_init_prio_data(cfqq);
        }
 
@@ -1407,7 +1398,7 @@ cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
 
        might_sleep_if(gfp_mask & __GFP_WAIT);
 
-       ioc = get_io_context(gfp_mask);
+       ioc = get_io_context(gfp_mask, cfqd->queue->node);
        if (!ioc)
                return NULL;
 
@@ -1671,12 +1662,8 @@ static void cfq_completed_request(request_queue_t *q, struct request *rq)
        if (!cfq_class_idle(cfqq))
                cfqd->last_end_request = now;
 
-       if (!cfq_cfqq_dispatched(cfqq)) {
-               if (cfq_cfqq_on_rr(cfqq)) {
-                       cfqq->service_last = now;
-                       cfq_resort_rr_list(cfqq, 0);
-               }
-       }
+       if (!cfq_cfqq_dispatched(cfqq) && cfq_cfqq_on_rr(cfqq))
+               cfq_resort_rr_list(cfqq, 0);
 
        if (sync)
                RQ_CIC(rq)->last_end_request = now;
@@ -1955,7 +1942,7 @@ static void *cfq_init_queue(request_queue_t *q, elevator_t *e)
        struct cfq_data *cfqd;
        int i;
 
-       cfqd = kmalloc(sizeof(*cfqd), GFP_KERNEL);
+       cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL, q->node);
        if (!cfqd)
                return NULL;
 
@@ -1967,10 +1954,9 @@ static void *cfq_init_queue(request_queue_t *q, elevator_t *e)
        INIT_LIST_HEAD(&cfqd->busy_rr);
        INIT_LIST_HEAD(&cfqd->cur_rr);
        INIT_LIST_HEAD(&cfqd->idle_rr);
-       INIT_LIST_HEAD(&cfqd->empty_list);
        INIT_LIST_HEAD(&cfqd->cic_list);
 
-       cfqd->cfq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_QHASH_ENTRIES, GFP_KERNEL);
+       cfqd->cfq_hash = kmalloc_node(sizeof(struct hlist_head) * CFQ_QHASH_ENTRIES, GFP_KERNEL, q->node);
        if (!cfqd->cfq_hash)
                goto out_free;