Commit | Line | Data |
---|---|---|
805e021f CE |
1 | /* |
2 | * Copyright (c) 2009 Simon Wilkinson. All rights reserved. | |
3 | * | |
4 | * Redistribution and use in source and binary forms, with or without | |
5 | * modification, are permitted provided that the following conditions | |
6 | * are met: | |
7 | * 1. Redistributions of source code must retain the above copyright | |
8 | * notice, this list of conditions and the following disclaimer. | |
9 | * 2. Redistributions in binary form must reproduce the above copyright | |
10 | * notice, this list of conditions and the following disclaimer in the | |
11 | * documentation and/or other materials provided with the distribution. | |
12 | * | |
13 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR `AS IS'' AND ANY EXPRESS OR | |
14 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |
15 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |
16 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | |
17 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |
18 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
19 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
20 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
21 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | |
22 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
23 | */ | |
24 | ||
25 | /* | |
26 | * Background page copying | |
27 | * | |
28 | * In the Linux CM, we pull cached files in from disk by reading them into | |
29 | * a page backed by the disk file, then copying them into the relevant AFS | |
30 | * page. This is a syncronous operation, requiring us to wait until the | |
31 | * disk read is completed before the page copy can be performed. When we're | |
32 | * doing readahead with readpages(), it means that the readpages() call must | |
33 | * block until the readahead is complete, which somewhat defeats the point. | |
34 | * | |
35 | * This file implements a background queuing system for performing these | |
36 | * page copies. For each collection of pages requiring copying, a new | |
37 | * task is created by calling afs_pagecopy_init_task(). Every time | |
38 | * readpage() on the backing cache returns a page which is still locked, | |
39 | * afs_pagecopy_queue_page() can be called to queue up a background copy | |
40 | * of this page. queue_page() ensures that the new page is connected to | |
41 | * the current task structure, and that that task is on a locally implemented | |
42 | * work queue. | |
43 | * | |
44 | * The work queue is handled by a dedicated kernel thread (created by | |
45 | * afs_init_pagecopy() and destroyed with afs_shutdown_pagecopy() ). This | |
46 | * thread iterates on the queue, moving all pages that are unlocked to a | |
47 | * different list, and placing tasks with unlocked pages onto the kernel | |
48 | * work queue. Once it has run through all of the unlocked pages, it will | |
49 | * identify a still-locked page to sleep upon, and wait until that page is | |
50 | * unlocked. | |
51 | * | |
52 | * The final act of copying the pages is performed by a per-task job in the | |
53 | * kernel work queue (this allows us to use multiple processors on SMP systems) | |
54 | */ | |
55 | ||
56 | #include <afsconfig.h> | |
57 | #include "afs/param.h" | |
58 | ||
59 | #include <linux/pagemap.h> | |
60 | #include <linux/kthread.h> | |
61 | #include <linux/wait.h> | |
62 | #include <linux/workqueue.h> | |
63 | #include <linux/slab.h> | |
64 | ||
65 | static DECLARE_WAIT_QUEUE_HEAD (afs_pagecopy_wq); | |
66 | static spinlock_t afs_pagecopy_lock; | |
67 | static struct list_head afs_pagecopy_tasks; | |
68 | static struct task_struct * afs_pagecopy_thread_id; | |
69 | ||
70 | struct afs_pagecopy_page { | |
71 | struct page *afspage; | |
72 | struct page *cachepage; | |
73 | struct list_head tasklink; | |
74 | }; | |
75 | ||
76 | struct afs_pagecopy_task { | |
77 | struct work_struct work; | |
78 | struct list_head checkpages; | |
79 | struct list_head copypages; | |
80 | atomic_t refcnt; | |
81 | spinlock_t lock; | |
82 | struct list_head joblink; | |
83 | }; | |
84 | ||
85 | #if defined(INIT_WORK_HAS_DATA) | |
86 | static void afs_pagecopy_worker(void *rock); | |
87 | #else | |
88 | static void afs_pagecopy_worker(struct work_struct *work); | |
89 | #endif | |
90 | ||
91 | struct afs_pagecopy_task * | |
92 | afs_pagecopy_init_task(void) { | |
93 | struct afs_pagecopy_task *task; | |
94 | ||
95 | task = kzalloc(sizeof(struct afs_pagecopy_task), GFP_NOFS); | |
96 | INIT_LIST_HEAD(&task->checkpages); | |
97 | INIT_LIST_HEAD(&task->copypages); | |
98 | INIT_LIST_HEAD(&task->joblink); | |
99 | #if defined(INIT_WORK_HAS_DATA) | |
100 | INIT_WORK(&task->work, afs_pagecopy_worker, &task->work); | |
101 | #else | |
102 | INIT_WORK(&task->work, afs_pagecopy_worker); | |
103 | #endif | |
104 | spin_lock_init(&task->lock); | |
105 | atomic_inc(&task->refcnt); | |
106 | ||
107 | return task; | |
108 | } | |
109 | ||
110 | void afs_pagecopy_queue_page(struct afs_pagecopy_task *task, | |
111 | struct page *cachepage, | |
112 | struct page *afspage) | |
113 | { | |
114 | struct afs_pagecopy_page *page; | |
115 | ||
116 | page = kzalloc(sizeof(struct afs_pagecopy_page), GFP_NOFS); | |
117 | INIT_LIST_HEAD(&page->tasklink); | |
118 | ||
119 | get_page(cachepage); | |
120 | page->cachepage = cachepage; | |
121 | get_page(afspage); | |
122 | page->afspage = afspage; | |
123 | ||
124 | spin_lock(&task->lock); | |
125 | list_add_tail(&page->tasklink, &task->checkpages); | |
126 | spin_lock(&afs_pagecopy_lock); | |
127 | if (list_empty(&task->joblink)) { | |
128 | atomic_inc(&task->refcnt); | |
129 | list_add_tail(&task->joblink, &afs_pagecopy_tasks); | |
130 | } | |
131 | spin_unlock(&afs_pagecopy_lock); | |
132 | spin_unlock(&task->lock); | |
133 | ||
134 | wake_up_interruptible(&afs_pagecopy_wq); | |
135 | } | |
136 | ||
137 | void afs_pagecopy_put_task(struct afs_pagecopy_task *task) | |
138 | { | |
139 | if (!atomic_dec_and_test(&task->refcnt)) | |
140 | return; | |
141 | ||
142 | kfree(task); | |
143 | } | |
144 | ||
145 | static struct page * afs_pagecopy_checkworkload(void) { | |
146 | struct page *sleeppage = NULL; | |
147 | struct afs_pagecopy_task *task, *tmp_task; | |
148 | struct afs_pagecopy_page *page, *tmp_page; | |
149 | ||
150 | spin_lock(&afs_pagecopy_lock); | |
151 | list_for_each_entry_safe(task, tmp_task, &afs_pagecopy_tasks, joblink) { | |
152 | spin_unlock(&afs_pagecopy_lock); | |
153 | ||
154 | spin_lock(&task->lock); | |
155 | list_for_each_entry_safe(page, tmp_page, &task->checkpages, tasklink) { | |
156 | if (!PageLocked(page->cachepage)) { | |
157 | list_move_tail(&page->tasklink, &task->copypages); | |
158 | atomic_inc(&task->refcnt); | |
159 | if (!schedule_work(&task->work)) | |
160 | atomic_dec(&task->refcnt); | |
161 | } else if (!sleeppage) { | |
162 | get_page(page->cachepage); | |
163 | sleeppage = page->cachepage; | |
164 | } | |
165 | } | |
166 | /* If the task structure has no more pages to check, remove it | |
167 | * from our workload queue */ | |
168 | if (list_empty(&task->checkpages)) { | |
169 | spin_lock(&afs_pagecopy_lock); | |
170 | spin_unlock(&task->lock); | |
171 | list_del_init(&task->joblink); | |
172 | spin_unlock(&afs_pagecopy_lock); | |
173 | afs_pagecopy_put_task(task); | |
174 | } else { | |
175 | spin_unlock(&task->lock); | |
176 | } | |
177 | spin_lock(&afs_pagecopy_lock); | |
178 | } | |
179 | spin_unlock(&afs_pagecopy_lock); | |
180 | ||
181 | return sleeppage; | |
182 | } | |
183 | ||
184 | #if defined(INIT_WORK_HAS_DATA) | |
185 | static void afs_pagecopy_worker(void *work) | |
186 | #else | |
187 | static void afs_pagecopy_worker(struct work_struct *work) | |
188 | #endif | |
189 | { | |
190 | struct afs_pagecopy_task *task = | |
191 | container_of(work, struct afs_pagecopy_task, work); | |
192 | struct afs_pagecopy_page *page; | |
193 | ||
194 | spin_lock(&task->lock); | |
195 | while (!list_empty(&task->copypages)) { | |
196 | page = list_entry(task->copypages.next, struct afs_pagecopy_page, | |
197 | tasklink); | |
198 | list_del(&page->tasklink); | |
199 | spin_unlock(&task->lock); | |
200 | ||
201 | if (PageUptodate(page->cachepage)) { | |
202 | copy_highpage(page->afspage, page->cachepage); | |
203 | flush_dcache_page(page->afspage); | |
204 | ClearPageError(page->afspage); | |
205 | SetPageUptodate(page->afspage); | |
206 | } | |
207 | unlock_page(page->afspage); | |
208 | put_page(page->cachepage); | |
209 | put_page(page->afspage); | |
210 | kfree(page); | |
211 | ||
212 | spin_lock(&task->lock); | |
213 | } | |
214 | spin_unlock(&task->lock); | |
215 | ||
216 | afs_pagecopy_put_task(task); | |
217 | } | |
218 | ||
219 | static int afs_pagecopy_thread(void *unused) { | |
220 | struct page *sleeppage; | |
221 | ||
222 | while (!kthread_should_stop()) { | |
223 | for (;;) { | |
224 | sleeppage = afs_pagecopy_checkworkload(); | |
225 | if (sleeppage) { | |
226 | wait_on_page_locked(sleeppage); | |
227 | put_page(sleeppage); | |
228 | } else { | |
229 | break; | |
230 | } | |
231 | } | |
232 | wait_event_interruptible(afs_pagecopy_wq, | |
233 | !list_empty(&afs_pagecopy_tasks) || kthread_should_stop()); | |
234 | } | |
235 | ||
236 | return 0; | |
237 | } | |
238 | ||
239 | void afs_init_pagecopy(void) { | |
240 | spin_lock_init(&afs_pagecopy_lock); | |
241 | INIT_LIST_HEAD(&afs_pagecopy_tasks); | |
242 | ||
243 | afs_pagecopy_thread_id = kthread_run(afs_pagecopy_thread, NULL, | |
244 | "afs_pagecopy"); | |
245 | } | |
246 | ||
247 | void afs_shutdown_pagecopy(void) { | |
248 | if (afs_pagecopy_thread_id) | |
249 | kthread_stop(afs_pagecopy_thread_id); | |
250 | } | |
251 |