2 * Copyright (c) 2009 Simon Wilkinson. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR `AS IS'' AND ANY EXPRESS OR
14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * Background page copying
28 * In the Linux CM, we pull cached files in from disk by reading them into
29 * a page backed by the disk file, then copying them into the relevant AFS
30 * page. This is a syncronous operation, requiring us to wait until the
31 * disk read is completed before the page copy can be performed. When we're
32 * doing readahead with readpages(), it means that the readpages() call must
33 * block until the readahead is complete, which somewhat defeats the point.
35 * This file implements a background queuing system for performing these
36 * page copies. For each collection of pages requiring copying, a new
37 * task is created by calling afs_pagecopy_init_task(). Every time
38 * readpage() on the backing cache returns a page which is still locked,
39 * afs_pagecopy_queue_page() can be called to queue up a background copy
40 * of this page. queue_page() ensures that the new page is connected to
41 * the current task structure, and that that task is on a locally implemented
44 * The work queue is handled by a dedicated kernel thread (created by
45 * afs_init_pagecopy() and destroyed with afs_shutdown_pagecopy() ). This
46 * thread iterates on the queue, moving all pages that are unlocked to a
47 * different list, and placing tasks with unlocked pages onto the kernel
48 * work queue. Once it has run through all of the unlocked pages, it will
49 * identify a still-locked page to sleep upon, and wait until that page is
52 * The final act of copying the pages is performed by a per-task job in the
53 * kernel work queue (this allows us to use multiple processors on SMP systems)
56 #include <afsconfig.h>
57 #include "afs/param.h"
59 #include <linux/pagemap.h>
60 #include <linux/kthread.h>
61 #include <linux/wait.h>
62 #include <linux/workqueue.h>
63 #include <linux/slab.h>
65 static DECLARE_WAIT_QUEUE_HEAD (afs_pagecopy_wq
);
66 static spinlock_t afs_pagecopy_lock
;
67 static struct list_head afs_pagecopy_tasks
;
68 static struct task_struct
* afs_pagecopy_thread_id
;
70 struct afs_pagecopy_page
{
72 struct page
*cachepage
;
73 struct list_head tasklink
;
76 struct afs_pagecopy_task
{
77 struct work_struct work
;
78 struct list_head checkpages
;
79 struct list_head copypages
;
82 struct list_head joblink
;
85 #if defined(INIT_WORK_HAS_DATA)
86 static void afs_pagecopy_worker(void *rock
);
88 static void afs_pagecopy_worker(struct work_struct
*work
);
91 struct afs_pagecopy_task
*
92 afs_pagecopy_init_task(void) {
93 struct afs_pagecopy_task
*task
;
95 task
= kzalloc(sizeof(struct afs_pagecopy_task
), GFP_NOFS
);
96 INIT_LIST_HEAD(&task
->checkpages
);
97 INIT_LIST_HEAD(&task
->copypages
);
98 INIT_LIST_HEAD(&task
->joblink
);
99 #if defined(INIT_WORK_HAS_DATA)
100 INIT_WORK(&task
->work
, afs_pagecopy_worker
, &task
->work
);
102 INIT_WORK(&task
->work
, afs_pagecopy_worker
);
104 spin_lock_init(&task
->lock
);
105 atomic_inc(&task
->refcnt
);
110 void afs_pagecopy_queue_page(struct afs_pagecopy_task
*task
,
111 struct page
*cachepage
,
112 struct page
*afspage
)
114 struct afs_pagecopy_page
*page
;
116 page
= kzalloc(sizeof(struct afs_pagecopy_page
), GFP_NOFS
);
117 INIT_LIST_HEAD(&page
->tasklink
);
120 page
->cachepage
= cachepage
;
122 page
->afspage
= afspage
;
124 spin_lock(&task
->lock
);
125 list_add_tail(&page
->tasklink
, &task
->checkpages
);
126 spin_lock(&afs_pagecopy_lock
);
127 if (list_empty(&task
->joblink
)) {
128 atomic_inc(&task
->refcnt
);
129 list_add_tail(&task
->joblink
, &afs_pagecopy_tasks
);
131 spin_unlock(&afs_pagecopy_lock
);
132 spin_unlock(&task
->lock
);
134 wake_up_interruptible(&afs_pagecopy_wq
);
137 void afs_pagecopy_put_task(struct afs_pagecopy_task
*task
)
139 if (!atomic_dec_and_test(&task
->refcnt
))
145 static struct page
* afs_pagecopy_checkworkload(void) {
146 struct page
*sleeppage
= NULL
;
147 struct afs_pagecopy_task
*task
, *tmp_task
;
148 struct afs_pagecopy_page
*page
, *tmp_page
;
150 spin_lock(&afs_pagecopy_lock
);
151 list_for_each_entry_safe(task
, tmp_task
, &afs_pagecopy_tasks
, joblink
) {
152 spin_unlock(&afs_pagecopy_lock
);
154 spin_lock(&task
->lock
);
155 list_for_each_entry_safe(page
, tmp_page
, &task
->checkpages
, tasklink
) {
156 if (!PageLocked(page
->cachepage
)) {
157 list_move_tail(&page
->tasklink
, &task
->copypages
);
158 atomic_inc(&task
->refcnt
);
159 if (!schedule_work(&task
->work
))
160 atomic_dec(&task
->refcnt
);
161 } else if (!sleeppage
) {
162 get_page(page
->cachepage
);
163 sleeppage
= page
->cachepage
;
166 /* If the task structure has no more pages to check, remove it
167 * from our workload queue */
168 if (list_empty(&task
->checkpages
)) {
169 spin_lock(&afs_pagecopy_lock
);
170 spin_unlock(&task
->lock
);
171 list_del_init(&task
->joblink
);
172 spin_unlock(&afs_pagecopy_lock
);
173 afs_pagecopy_put_task(task
);
175 spin_unlock(&task
->lock
);
177 spin_lock(&afs_pagecopy_lock
);
179 spin_unlock(&afs_pagecopy_lock
);
184 #if defined(INIT_WORK_HAS_DATA)
185 static void afs_pagecopy_worker(void *work
)
187 static void afs_pagecopy_worker(struct work_struct
*work
)
190 struct afs_pagecopy_task
*task
=
191 container_of(work
, struct afs_pagecopy_task
, work
);
192 struct afs_pagecopy_page
*page
;
194 spin_lock(&task
->lock
);
195 while (!list_empty(&task
->copypages
)) {
196 page
= list_entry(task
->copypages
.next
, struct afs_pagecopy_page
,
198 list_del(&page
->tasklink
);
199 spin_unlock(&task
->lock
);
201 if (PageUptodate(page
->cachepage
)) {
202 copy_highpage(page
->afspage
, page
->cachepage
);
203 flush_dcache_page(page
->afspage
);
204 ClearPageError(page
->afspage
);
205 SetPageUptodate(page
->afspage
);
207 unlock_page(page
->afspage
);
208 put_page(page
->cachepage
);
209 put_page(page
->afspage
);
212 spin_lock(&task
->lock
);
214 spin_unlock(&task
->lock
);
216 afs_pagecopy_put_task(task
);
219 static int afs_pagecopy_thread(void *unused
) {
220 struct page
*sleeppage
;
222 while (!kthread_should_stop()) {
224 sleeppage
= afs_pagecopy_checkworkload();
226 wait_on_page_locked(sleeppage
);
232 wait_event_interruptible(afs_pagecopy_wq
,
233 !list_empty(&afs_pagecopy_tasks
) || kthread_should_stop());
239 void afs_init_pagecopy(void) {
240 spin_lock_init(&afs_pagecopy_lock
);
241 INIT_LIST_HEAD(&afs_pagecopy_tasks
);
243 afs_pagecopy_thread_id
= kthread_run(afs_pagecopy_thread
, NULL
,
247 void afs_shutdown_pagecopy(void) {
248 if (afs_pagecopy_thread_id
)
249 kthread_stop(afs_pagecopy_thread_id
);