src/afs/LINUX/osi_pagecopy.c

   1 /*
   2  * Copyright (c) 2009 Simon Wilkinson. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR `AS IS'' AND ANY EXPRESS OR
  14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 /*
  26  * Background page copying
  27  *
  28  * In the Linux CM, we pull cached files in from disk by reading them into
  29  * a page backed by the disk file, then copying them into the relevant AFS
  30  * page. This is a syncronous operation, requiring us to wait until the
  31  * disk read is completed before the page copy can be performed. When we're
  32  * doing readahead with readpages(), it means that the readpages() call must
  33  * block until the readahead is complete, which somewhat defeats the point.
  34  *
  35  * This file implements a background queuing system for performing these
  36  * page copies. For each collection of pages requiring copying, a new
  37  * task is created by calling afs_pagecopy_init_task(). Every time
  38  * readpage() on the backing cache returns a page which is still locked,
  39  * afs_pagecopy_queue_page() can be called to queue up a background copy
  40  * of this page.  queue_page() ensures that the new page is connected to
  41  * the current task structure, and that that task is on a locally implemented
  42  * work queue.
  43  *
  44  * The work queue is handled by a dedicated kernel thread (created by
  45  * afs_init_pagecopy() and destroyed with afs_shutdown_pagecopy() ). This
  46  * thread iterates on the queue, moving all pages that are unlocked to a
  47  * different list, and placing tasks with unlocked pages onto the kernel
  48  * work queue. Once it has run through all of the unlocked pages, it will
  49  * identify a still-locked page to sleep upon, and wait until that page is
  50  * unlocked.
  51  *
  52  * The final act of copying the pages is performed by a per-task job in the
  53  * kernel work queue (this allows us to use multiple processors on SMP systems)
  54  */
  55
  56 #include <afsconfig.h>
  57 #include "afs/param.h"
  58
  59 #include <linux/pagemap.h>
  60 #include <linux/kthread.h>
  61 #include <linux/wait.h>
  62 #include <linux/workqueue.h>
  63 #include <linux/slab.h>
  64
  65 static DECLARE_WAIT_QUEUE_HEAD (afs_pagecopy_wq);
  66 static spinlock_t afs_pagecopy_lock;
  67 static struct list_head afs_pagecopy_tasks;
  68 static struct task_struct * afs_pagecopy_thread_id;
  69
  70 struct afs_pagecopy_page {
  71     struct page *afspage;
  72     struct page *cachepage;
  73     struct list_head tasklink;
  74 };
  75
  76 struct afs_pagecopy_task {
  77     struct work_struct work;
  78     struct list_head checkpages;
  79     struct list_head copypages;
  80     atomic_t refcnt;
  81     spinlock_t lock;
  82     struct list_head joblink;
  83 };
  84
  85 #if defined(INIT_WORK_HAS_DATA)
  86 static void afs_pagecopy_worker(void *rock);
  87 #else
  88 static void afs_pagecopy_worker(struct work_struct *work);
  89 #endif
  90
  91 struct afs_pagecopy_task *
  92 afs_pagecopy_init_task(void) {
  93     struct afs_pagecopy_task *task;
  94
  95     task = kzalloc(sizeof(struct afs_pagecopy_task), GFP_NOFS);
  96     INIT_LIST_HEAD(&task->checkpages);
  97     INIT_LIST_HEAD(&task->copypages);
  98     INIT_LIST_HEAD(&task->joblink);
  99 #if defined(INIT_WORK_HAS_DATA)
 100     INIT_WORK(&task->work, afs_pagecopy_worker, &task->work);
 101 #else
 102     INIT_WORK(&task->work, afs_pagecopy_worker);
 103 #endif
 104     spin_lock_init(&task->lock);
 105     atomic_inc(&task->refcnt);
 106
 107     return task;
 108 }
 109
 110 void afs_pagecopy_queue_page(struct afs_pagecopy_task *task,
 111                              struct page *cachepage,
 112                              struct page *afspage)
 113 {
 114     struct afs_pagecopy_page *page;
 115
 116     page = kzalloc(sizeof(struct afs_pagecopy_page), GFP_NOFS);
 117     INIT_LIST_HEAD(&page->tasklink);
 118
 119     get_page(cachepage);
 120     page->cachepage = cachepage;
 121     get_page(afspage);
 122     page->afspage = afspage;
 123
 124     spin_lock(&task->lock);
 125     list_add_tail(&page->tasklink, &task->checkpages);
 126     spin_lock(&afs_pagecopy_lock);
 127     if (list_empty(&task->joblink)) {
 128         atomic_inc(&task->refcnt);
 129         list_add_tail(&task->joblink, &afs_pagecopy_tasks);
 130     }
 131     spin_unlock(&afs_pagecopy_lock);
 132     spin_unlock(&task->lock);
 133
 134     wake_up_interruptible(&afs_pagecopy_wq);
 135 }
 136
 137 void afs_pagecopy_put_task(struct afs_pagecopy_task *task)
 138 {
 139     if (!atomic_dec_and_test(&task->refcnt))
 140         return;
 141
 142     kfree(task);
 143 }
 144
 145 static struct page * afs_pagecopy_checkworkload(void) {
 146     struct page *sleeppage = NULL;
 147     struct afs_pagecopy_task *task, *tmp_task;
 148     struct afs_pagecopy_page *page, *tmp_page;
 149
 150     spin_lock(&afs_pagecopy_lock);
 151     list_for_each_entry_safe(task, tmp_task, &afs_pagecopy_tasks, joblink) {
 152         spin_unlock(&afs_pagecopy_lock);
 153
 154         spin_lock(&task->lock);
 155         list_for_each_entry_safe(page, tmp_page, &task->checkpages, tasklink) {
 156            if (!PageLocked(page->cachepage)) {
 157                 list_move_tail(&page->tasklink, &task->copypages);
 158                 atomic_inc(&task->refcnt);
 159                 if (!schedule_work(&task->work))
 160                    atomic_dec(&task->refcnt);
 161            } else if (!sleeppage) {
 162                 get_page(page->cachepage);
 163                 sleeppage = page->cachepage;
 164            }
 165         }
 166         /* If the task structure has no more pages to check, remove it
 167          * from our workload queue */
 168         if (list_empty(&task->checkpages)) {
 169             spin_lock(&afs_pagecopy_lock);
 170             spin_unlock(&task->lock);
 171             list_del_init(&task->joblink);
 172             spin_unlock(&afs_pagecopy_lock);
 173             afs_pagecopy_put_task(task);
 174         } else {
 175             spin_unlock(&task->lock);
 176         }
 177         spin_lock(&afs_pagecopy_lock);
 178     }
 179     spin_unlock(&afs_pagecopy_lock);
 180
 181     return sleeppage;
 182 }
 183
 184 #if defined(INIT_WORK_HAS_DATA)
 185 static void afs_pagecopy_worker(void *work)
 186 #else
 187 static void afs_pagecopy_worker(struct work_struct *work)
 188 #endif
 189 {
 190     struct afs_pagecopy_task *task =
 191         container_of(work, struct afs_pagecopy_task, work);
 192     struct afs_pagecopy_page *page;
 193
 194     spin_lock(&task->lock);
 195     while (!list_empty(&task->copypages)) {
 196         page = list_entry(task->copypages.next, struct afs_pagecopy_page,
 197                           tasklink);
 198         list_del(&page->tasklink);
 199         spin_unlock(&task->lock);
 200
 201         if (PageUptodate(page->cachepage)) {
 202             copy_highpage(page->afspage, page->cachepage);
 203             flush_dcache_page(page->afspage);
 204             ClearPageError(page->afspage);
 205             SetPageUptodate(page->afspage);
 206         }
 207         unlock_page(page->afspage);
 208         put_page(page->cachepage);
 209         put_page(page->afspage);
 210         kfree(page);
 211
 212         spin_lock(&task->lock);
 213     }
 214     spin_unlock(&task->lock);
 215
 216     afs_pagecopy_put_task(task);
 217 }
 218
 219 static int afs_pagecopy_thread(void *unused) {
 220     struct page *sleeppage;
 221
 222     while (!kthread_should_stop()) {
 223         for (;;) {
 224             sleeppage = afs_pagecopy_checkworkload();
 225             if (sleeppage) {
 226                 wait_on_page_locked(sleeppage);
 227                 put_page(sleeppage);
 228             } else {
 229                 break;
 230             }
 231         }
 232         wait_event_interruptible(afs_pagecopy_wq,
 233                    !list_empty(&afs_pagecopy_tasks) || kthread_should_stop());
 234     }
 235
 236     return 0;
 237 }
 238
 239 void afs_init_pagecopy(void) {
 240    spin_lock_init(&afs_pagecopy_lock);
 241    INIT_LIST_HEAD(&afs_pagecopy_tasks);
 242
 243    afs_pagecopy_thread_id = kthread_run(afs_pagecopy_thread, NULL,
 244                                         "afs_pagecopy");
 245 }
 246
 247 void afs_shutdown_pagecopy(void) {
 248    if (afs_pagecopy_thread_id)
 249         kthread_stop(afs_pagecopy_thread_id);
 250 }
 251