Commit | Line | Data |
---|---|---|
805e021f CE |
1 | /* |
2 | * Copyright 2000, International Business Machines Corporation and others. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * This software has been released under the terms of the IBM Public | |
6 | * License. For details, see the LICENSE file in the top-level source | |
7 | * directory or online at http://www.openafs.org/dl/license10.html | |
8 | */ | |
9 | ||
10 | /* | |
11 | * osi_alloc.c - Linux memory allocation routines. | |
12 | * | |
13 | */ | |
14 | #include <afsconfig.h> | |
15 | #include "afs/param.h" | |
16 | ||
17 | ||
18 | #include "afs/sysincludes.h" | |
19 | #include "afsincludes.h" | |
20 | #include <linux/mm.h> | |
21 | #include <linux/slab.h> | |
22 | ||
23 | #include "afs_atomlist.h" | |
24 | #include "afs_lhash.h" | |
25 | ||
26 | #define MAX_KMALLOC_SIZE PAGE_SIZE /* Max we should alloc with kmalloc */ | |
27 | ||
28 | /* types of alloc */ | |
29 | #define KM_TYPE 1 /* kmalloc */ | |
30 | #define VM_TYPE 2 /* vmalloc */ | |
31 | ||
32 | struct osi_linux_mem { | |
33 | void *chunk; | |
34 | }; | |
35 | ||
36 | /* These assume 32-bit pointers */ | |
37 | #define MEMTYPE(A) (((unsigned long)A) & 0x3) | |
38 | #define MEMADDR(A) (void *)((unsigned long)(A) & (~0x3)) | |
39 | ||
40 | /* globals */ | |
41 | afs_atomlist *al_mem_pool; /* pool of osi_linux_mem structures */ | |
42 | afs_lhash *lh_mem_htab; /* mem hash table */ | |
43 | unsigned int allocator_init = 0; /* has the allocator been initialized? */ | |
44 | unsigned int afs_linux_cur_allocs = 0; | |
45 | unsigned int afs_linux_total_allocs = 0; | |
46 | unsigned int afs_linux_hash_verify_count = 0; /* used by hash_verify */ | |
47 | ||
48 | #include <linux/vmalloc.h> | |
49 | ||
50 | /* Allocator support functions (static) */ | |
51 | ||
52 | static int | |
53 | hash_equal(const void *a, const void *b) | |
54 | { | |
55 | return (MEMADDR(((struct osi_linux_mem *)a)->chunk) == | |
56 | MEMADDR(((struct osi_linux_mem *)b)->chunk)); | |
57 | ||
58 | } | |
59 | ||
60 | /* linux_alloc : Allocates memory from the linux kernel. It uses | |
61 | * kmalloc if possible. Otherwise, we use vmalloc. | |
62 | * Input: | |
63 | * asize - size of memory required in bytes | |
64 | * Return Values: | |
65 | * returns NULL if we failed to allocate memory. | |
66 | * or pointer to memory if we succeeded. | |
67 | */ | |
68 | static void * | |
69 | linux_alloc(unsigned int asize, int drop_glock) | |
70 | { | |
71 | void *new = NULL; | |
72 | int max_retry = 10; | |
73 | int haveGlock = ISAFS_GLOCK(); | |
74 | ||
75 | /* if we can use kmalloc use it to allocate the required memory. */ | |
76 | while (!new && max_retry) { | |
77 | if (asize <= MAX_KMALLOC_SIZE) { | |
78 | new = (void *)(unsigned long)kmalloc(asize, GFP_NOFS); | |
79 | if (new) /* piggy back alloc type */ | |
80 | new = (void *)(KM_TYPE | (unsigned long)new); | |
81 | } else { | |
82 | osi_Assert(drop_glock || !haveGlock); | |
83 | if (drop_glock && haveGlock) | |
84 | AFS_GUNLOCK(); | |
85 | new = (void *)vmalloc(asize); | |
86 | if (drop_glock && haveGlock) | |
87 | AFS_GLOCK(); | |
88 | if (new) /* piggy back alloc type */ | |
89 | new = (void *)(VM_TYPE | (unsigned long)new); | |
90 | } | |
91 | ||
92 | if (!new) { | |
93 | #ifdef set_current_state | |
94 | set_current_state(TASK_INTERRUPTIBLE); | |
95 | #else | |
96 | current->state = TASK_INTERRUPTIBLE; | |
97 | #endif | |
98 | if (drop_glock && haveGlock) | |
99 | AFS_GUNLOCK(); | |
100 | schedule_timeout(HZ); | |
101 | if (drop_glock && haveGlock) | |
102 | AFS_GLOCK(); | |
103 | #ifdef set_current_state | |
104 | set_current_state(TASK_RUNNING); | |
105 | #else | |
106 | current->state = TASK_RUNNING; | |
107 | #endif | |
108 | --max_retry; | |
109 | } | |
110 | } | |
111 | if (new) | |
112 | memset(MEMADDR(new), 0, asize); | |
113 | ||
114 | return new; | |
115 | } | |
116 | ||
117 | static void | |
118 | linux_free(void *p) | |
119 | { | |
120 | ||
121 | /* mask out the type information from the pointer and | |
122 | * use the appropriate free routine to free the chunk. | |
123 | */ | |
124 | switch (MEMTYPE(p)) { | |
125 | case KM_TYPE: | |
126 | kfree(MEMADDR(p)); | |
127 | break; | |
128 | case VM_TYPE: | |
129 | vfree(MEMADDR(p)); | |
130 | break; | |
131 | default: | |
132 | printf("afs_osi_Free: Asked to free unknown type %d at 0x%lx\n", | |
133 | (int)MEMTYPE(p), (unsigned long)MEMADDR(p)); | |
134 | break; | |
135 | } | |
136 | ||
137 | } | |
138 | ||
139 | /* hash_chunk() receives a pointer to a chunk and hashes it to produce a | |
140 | * key that the hashtable can use. The key is obtained by | |
141 | * right shifting out the 2 LSBs and then multiplying the | |
142 | * result by a constant no. and dividing it with a large prime. | |
143 | */ | |
144 | #define HASH_CONST 32786 | |
145 | #define HASH_PRIME 79367 | |
146 | static unsigned | |
147 | hash_chunk(void *p) | |
148 | { | |
149 | unsigned int key; | |
150 | ||
151 | key = (unsigned int)(long)p >> 2; | |
152 | key = (key * HASH_CONST) % HASH_PRIME; | |
153 | ||
154 | return key; | |
155 | } | |
156 | ||
157 | /* hash_free() : Invoked by osi_linux_free_afs_memory(), thru | |
158 | * afs_lhash_iter(), this function is called by the lhash | |
159 | * module for every entry in the hash table. hash_free | |
160 | * frees the memory associated with the entry as well | |
161 | * as returning the osi_linux_mem struct to its pool. | |
162 | */ | |
163 | static void | |
164 | hash_free(size_t index, unsigned key, void *data) | |
165 | { | |
166 | linux_free(((struct osi_linux_mem *)data)->chunk); | |
167 | afs_atomlist_put(al_mem_pool, data); | |
168 | } | |
169 | ||
170 | /* hash_verify() is invoked by osi_linux_verify_alloced_memory() thru | |
171 | * afs_lhash_iter() and is called by the lhash module for every element | |
172 | * in the hash table. | |
173 | * hash_verify() verifies (within limits) that the memory passed to it is | |
174 | * valid. | |
175 | */ | |
176 | static void | |
177 | hash_verify(size_t index, unsigned key, void *data) | |
178 | { | |
179 | struct osi_linux_mem *lmp = (struct osi_linux_mem *)data; | |
180 | int memtype; | |
181 | ||
182 | memtype = MEMTYPE(lmp->chunk); | |
183 | if (memtype != KM_TYPE && memtype != VM_TYPE) { | |
184 | printf | |
185 | ("osi_linux_verify_alloced_memory: unknown type %d at 0x%lx, index=%lu\n", | |
186 | (int)memtype, (unsigned long)lmp->chunk, (unsigned long)index); | |
187 | } | |
188 | afs_linux_hash_verify_count++; | |
189 | } | |
190 | ||
191 | ||
192 | /* local_free() : wrapper for vfree(), to deal with incompatible protoypes */ | |
193 | static void | |
194 | local_free(void *p, size_t n) | |
195 | { | |
196 | vfree(p); | |
197 | } | |
198 | ||
199 | /* linux_alloc_init(): Initializes the kernel memory allocator. As part | |
200 | * of this process, it also initializes a pool of osi_linux_mem | |
201 | * structures as well as the hash table itself. | |
202 | * Return values: | |
203 | * 0 - failure | |
204 | * 1 - success | |
205 | */ | |
206 | static int | |
207 | linux_alloc_init(void) | |
208 | { | |
209 | /* initiate our pool of osi_linux_mem structs */ | |
210 | al_mem_pool = | |
211 | afs_atomlist_create(sizeof(struct osi_linux_mem), sizeof(long) * 1024, | |
212 | (void *)vmalloc, local_free); | |
213 | if (!al_mem_pool) { | |
214 | printf("afs_osi_Alloc: Error in initialization(atomlist_create)\n"); | |
215 | return 0; | |
216 | } | |
217 | ||
218 | /* initialize the hash table to hold references to alloc'ed chunks */ | |
219 | lh_mem_htab = afs_lhash_create(hash_equal, (void *)vmalloc, local_free); | |
220 | if (!lh_mem_htab) { | |
221 | printf("afs_osi_Alloc: Error in initialization(lhash_create)\n"); | |
222 | return 0; | |
223 | } | |
224 | ||
225 | return 1; | |
226 | ||
227 | } | |
228 | ||
229 | /************** Linux memory allocator interface functions **********/ | |
230 | ||
231 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,16) | |
232 | DEFINE_MUTEX(afs_linux_alloc_sem); | |
233 | #else | |
234 | DECLARE_MUTEX(afs_linux_alloc_sem); | |
235 | #endif | |
236 | ||
237 | void * | |
238 | osi_linux_alloc(unsigned int asize, int drop_glock) | |
239 | { | |
240 | void *new = NULL; | |
241 | struct osi_linux_mem *lmem; | |
242 | ||
243 | new = linux_alloc(asize, drop_glock); /* get a chunk of memory of size asize */ | |
244 | ||
245 | if (!new) { | |
246 | printf("afs_osi_Alloc: Can't vmalloc %d bytes.\n", asize); | |
247 | return new; | |
248 | } | |
249 | ||
250 | mutex_lock(&afs_linux_alloc_sem); | |
251 | ||
252 | /* allocator hasn't been initialized yet */ | |
253 | if (allocator_init == 0) { | |
254 | if (linux_alloc_init() == 0) { | |
255 | goto error; | |
256 | } | |
257 | allocator_init = 1; /* initialization complete */ | |
258 | } | |
259 | ||
260 | /* get an atom to store the pointer to the chunk */ | |
261 | lmem = (struct osi_linux_mem *)afs_atomlist_get(al_mem_pool); | |
262 | if (!lmem) { | |
263 | printf("afs_osi_Alloc: atomlist_get() failed."); | |
264 | goto free_error; | |
265 | } | |
266 | /* store the chunk reference */ | |
267 | lmem->chunk = new; | |
268 | ||
269 | /* hash in the chunk */ | |
270 | if (afs_lhash_enter(lh_mem_htab, hash_chunk(new), lmem) != 0) { | |
271 | printf("afs_osi_Alloc: lhash_enter failed\n"); | |
272 | goto free_error; | |
273 | } | |
274 | afs_linux_cur_allocs++; /* no. of current allocations */ | |
275 | afs_linux_total_allocs++; /* total no. of allocations done so far */ | |
276 | error: | |
277 | mutex_unlock(&afs_linux_alloc_sem); | |
278 | return MEMADDR(new); | |
279 | ||
280 | free_error: | |
281 | if (new) { | |
282 | linux_free(new); | |
283 | } | |
284 | new = NULL; | |
285 | goto error; | |
286 | ||
287 | ||
288 | } | |
289 | ||
290 | /* osi_linux_free() - free chunk of memory passed to us. | |
291 | */ | |
292 | void | |
293 | osi_linux_free(void *addr) | |
294 | { | |
295 | struct osi_linux_mem lmem, *lmp; | |
296 | ||
297 | mutex_lock(&afs_linux_alloc_sem); | |
298 | ||
299 | lmem.chunk = addr; | |
300 | /* remove this chunk from our hash table */ | |
301 | if ((lmp = | |
302 | (struct osi_linux_mem *)afs_lhash_remove(lh_mem_htab, | |
303 | hash_chunk(addr), &lmem))) { | |
304 | linux_free(lmp->chunk); /* this contains the piggybacked type info */ | |
305 | afs_atomlist_put(al_mem_pool, lmp); /* return osi_linux_mem struct to pool */ | |
306 | afs_linux_cur_allocs--; | |
307 | } else { | |
308 | printf("osi_linux_free: failed to remove chunk from hashtable\n"); | |
309 | BUG(); | |
310 | } | |
311 | ||
312 | mutex_unlock(&afs_linux_alloc_sem); | |
313 | } | |
314 | ||
315 | /* osi_linux_free_afs_memory() - free all chunks of memory allocated. | |
316 | */ | |
317 | void | |
318 | osi_linux_free_afs_memory(void) | |
319 | { | |
320 | mutex_lock(&afs_linux_alloc_sem); | |
321 | ||
322 | if (allocator_init) { | |
323 | /* iterate through all elements in the hash table and free both | |
324 | * the chunk and the atom associated with it. | |
325 | */ | |
326 | afs_lhash_iter(lh_mem_htab, hash_free); | |
327 | ||
328 | /* free the atomlist. */ | |
329 | afs_atomlist_destroy(al_mem_pool); | |
330 | ||
331 | /* free the hashlist. */ | |
332 | afs_lhash_destroy(lh_mem_htab); | |
333 | ||
334 | /* change the state so that the allocator is now uninitialized. */ | |
335 | allocator_init = 0; | |
336 | } | |
337 | mutex_unlock(&afs_linux_alloc_sem); | |
338 | } | |
339 | ||
340 | /* osi_linux_verify_alloced_memory(): verify all chunks of alloced memory in | |
341 | * our hash table. | |
342 | */ | |
343 | void | |
344 | osi_linux_verify_alloced_memory() | |
345 | { | |
346 | mutex_lock(&afs_linux_alloc_sem); | |
347 | ||
348 | /* count of times hash_verify was called. reset it to 0 before iteration */ | |
349 | afs_linux_hash_verify_count = 0; | |
350 | ||
351 | /* iterate thru elements in the hash table */ | |
352 | afs_lhash_iter(lh_mem_htab, hash_verify); | |
353 | ||
354 | if (afs_linux_hash_verify_count != afs_linux_cur_allocs) { | |
355 | /* hmm, some pieces of memory are missing. */ | |
356 | printf | |
357 | ("osi_linux_verify_alloced_memory: %d chunks of memory are not accounted for during verify!\n", | |
358 | afs_linux_hash_verify_count - afs_linux_cur_allocs); | |
359 | } | |
360 | ||
361 | mutex_unlock(&afs_linux_alloc_sem); | |
362 | return; | |
363 | } | |
364 | ||
365 | #ifdef AFS_PRIVATE_OSI_ALLOCSPACES | |
366 | ||
367 | void | |
368 | osi_FreeLargeSpace(void *p) | |
369 | { | |
370 | kfree(p); | |
371 | } | |
372 | ||
373 | void | |
374 | osi_FreeSmallSpace(void *p) | |
375 | { | |
376 | kfree(p); | |
377 | } | |
378 | ||
379 | void * | |
380 | osi_AllocLargeSpace(size_t size) | |
381 | { | |
382 | if (size > AFS_LRALLOCSIZ) | |
383 | osi_Panic("osi_AllocLargeSpace: size=%d\n", (int) size); | |
384 | return kmalloc(AFS_LRALLOCSIZ, GFP_NOFS); | |
385 | } | |
386 | ||
387 | void * | |
388 | osi_AllocSmallSpace(size_t size) | |
389 | { | |
390 | if (size > AFS_SMALLOCSIZ) | |
391 | osi_Panic("osi_AllocSmallS: size=%d\n", (int)size); | |
392 | return kmalloc(AFS_SMALLOCSIZ, GFP_NOFS); | |
393 | } | |
394 | #endif /* AFS_PRIVATE_OSI_ALLOCSPACES */ |