| 1 | /* |
| 2 | * Copyright 2000, International Business Machines Corporation and others. |
| 3 | * All Rights Reserved. |
| 4 | * |
| 5 | * This software has been released under the terms of the IBM Public |
| 6 | * License. For details, see the LICENSE file in the top-level source |
| 7 | * directory or online at http://www.openafs.org/dl/license10.html |
| 8 | */ |
| 9 | |
| 10 | /* |
| 11 | * osi_alloc.c - Linux memory allocation routines. |
| 12 | * |
| 13 | */ |
| 14 | #include <afsconfig.h> |
| 15 | #include "afs/param.h" |
| 16 | |
| 17 | |
| 18 | #include "afs/sysincludes.h" |
| 19 | #include "afsincludes.h" |
| 20 | #include <linux/mm.h> |
| 21 | #include <linux/slab.h> |
| 22 | |
| 23 | #include "afs_atomlist.h" |
| 24 | #include "afs_lhash.h" |
| 25 | |
| 26 | #define MAX_KMALLOC_SIZE PAGE_SIZE /* Max we should alloc with kmalloc */ |
| 27 | |
| 28 | /* types of alloc */ |
| 29 | #define KM_TYPE 1 /* kmalloc */ |
| 30 | #define VM_TYPE 2 /* vmalloc */ |
| 31 | |
| 32 | struct osi_linux_mem { |
| 33 | void *chunk; |
| 34 | }; |
| 35 | |
| 36 | /* These assume 32-bit pointers */ |
| 37 | #define MEMTYPE(A) (((unsigned long)A) & 0x3) |
| 38 | #define MEMADDR(A) (void *)((unsigned long)(A) & (~0x3)) |
| 39 | |
| 40 | /* globals */ |
| 41 | afs_atomlist *al_mem_pool; /* pool of osi_linux_mem structures */ |
| 42 | afs_lhash *lh_mem_htab; /* mem hash table */ |
| 43 | unsigned int allocator_init = 0; /* has the allocator been initialized? */ |
| 44 | unsigned int afs_linux_cur_allocs = 0; |
| 45 | unsigned int afs_linux_total_allocs = 0; |
| 46 | unsigned int afs_linux_hash_verify_count = 0; /* used by hash_verify */ |
| 47 | |
| 48 | #include <linux/vmalloc.h> |
| 49 | |
| 50 | /* Allocator support functions (static) */ |
| 51 | |
| 52 | static int |
| 53 | hash_equal(const void *a, const void *b) |
| 54 | { |
| 55 | return (MEMADDR(((struct osi_linux_mem *)a)->chunk) == |
| 56 | MEMADDR(((struct osi_linux_mem *)b)->chunk)); |
| 57 | |
| 58 | } |
| 59 | |
| 60 | /* linux_alloc : Allocates memory from the linux kernel. It uses |
| 61 | * kmalloc if possible. Otherwise, we use vmalloc. |
| 62 | * Input: |
| 63 | * asize - size of memory required in bytes |
| 64 | * Return Values: |
| 65 | * returns NULL if we failed to allocate memory. |
| 66 | * or pointer to memory if we succeeded. |
| 67 | */ |
| 68 | static void * |
| 69 | linux_alloc(unsigned int asize, int drop_glock) |
| 70 | { |
| 71 | void *new = NULL; |
| 72 | int max_retry = 10; |
| 73 | int haveGlock = ISAFS_GLOCK(); |
| 74 | |
| 75 | /* if we can use kmalloc use it to allocate the required memory. */ |
| 76 | while (!new && max_retry) { |
| 77 | if (asize <= MAX_KMALLOC_SIZE) { |
| 78 | new = (void *)(unsigned long)kmalloc(asize, GFP_NOFS); |
| 79 | if (new) /* piggy back alloc type */ |
| 80 | new = (void *)(KM_TYPE | (unsigned long)new); |
| 81 | } else { |
| 82 | osi_Assert(drop_glock || !haveGlock); |
| 83 | if (drop_glock && haveGlock) |
| 84 | AFS_GUNLOCK(); |
| 85 | new = (void *)vmalloc(asize); |
| 86 | if (drop_glock && haveGlock) |
| 87 | AFS_GLOCK(); |
| 88 | if (new) /* piggy back alloc type */ |
| 89 | new = (void *)(VM_TYPE | (unsigned long)new); |
| 90 | } |
| 91 | |
| 92 | if (!new) { |
| 93 | #ifdef set_current_state |
| 94 | set_current_state(TASK_INTERRUPTIBLE); |
| 95 | #else |
| 96 | current->state = TASK_INTERRUPTIBLE; |
| 97 | #endif |
| 98 | if (drop_glock && haveGlock) |
| 99 | AFS_GUNLOCK(); |
| 100 | schedule_timeout(HZ); |
| 101 | if (drop_glock && haveGlock) |
| 102 | AFS_GLOCK(); |
| 103 | #ifdef set_current_state |
| 104 | set_current_state(TASK_RUNNING); |
| 105 | #else |
| 106 | current->state = TASK_RUNNING; |
| 107 | #endif |
| 108 | --max_retry; |
| 109 | } |
| 110 | } |
| 111 | if (new) |
| 112 | memset(MEMADDR(new), 0, asize); |
| 113 | |
| 114 | return new; |
| 115 | } |
| 116 | |
| 117 | static void |
| 118 | linux_free(void *p) |
| 119 | { |
| 120 | |
| 121 | /* mask out the type information from the pointer and |
| 122 | * use the appropriate free routine to free the chunk. |
| 123 | */ |
| 124 | switch (MEMTYPE(p)) { |
| 125 | case KM_TYPE: |
| 126 | kfree(MEMADDR(p)); |
| 127 | break; |
| 128 | case VM_TYPE: |
| 129 | vfree(MEMADDR(p)); |
| 130 | break; |
| 131 | default: |
| 132 | printf("afs_osi_Free: Asked to free unknown type %d at 0x%lx\n", |
| 133 | (int)MEMTYPE(p), (unsigned long)MEMADDR(p)); |
| 134 | break; |
| 135 | } |
| 136 | |
| 137 | } |
| 138 | |
| 139 | /* hash_chunk() receives a pointer to a chunk and hashes it to produce a |
| 140 | * key that the hashtable can use. The key is obtained by |
| 141 | * right shifting out the 2 LSBs and then multiplying the |
| 142 | * result by a constant no. and dividing it with a large prime. |
| 143 | */ |
| 144 | #define HASH_CONST 32786 |
| 145 | #define HASH_PRIME 79367 |
| 146 | static unsigned |
| 147 | hash_chunk(void *p) |
| 148 | { |
| 149 | unsigned int key; |
| 150 | |
| 151 | key = (unsigned int)(long)p >> 2; |
| 152 | key = (key * HASH_CONST) % HASH_PRIME; |
| 153 | |
| 154 | return key; |
| 155 | } |
| 156 | |
| 157 | /* hash_free() : Invoked by osi_linux_free_afs_memory(), thru |
| 158 | * afs_lhash_iter(), this function is called by the lhash |
| 159 | * module for every entry in the hash table. hash_free |
| 160 | * frees the memory associated with the entry as well |
| 161 | * as returning the osi_linux_mem struct to its pool. |
| 162 | */ |
| 163 | static void |
| 164 | hash_free(size_t index, unsigned key, void *data) |
| 165 | { |
| 166 | linux_free(((struct osi_linux_mem *)data)->chunk); |
| 167 | afs_atomlist_put(al_mem_pool, data); |
| 168 | } |
| 169 | |
| 170 | /* hash_verify() is invoked by osi_linux_verify_alloced_memory() thru |
| 171 | * afs_lhash_iter() and is called by the lhash module for every element |
| 172 | * in the hash table. |
| 173 | * hash_verify() verifies (within limits) that the memory passed to it is |
| 174 | * valid. |
| 175 | */ |
| 176 | static void |
| 177 | hash_verify(size_t index, unsigned key, void *data) |
| 178 | { |
| 179 | struct osi_linux_mem *lmp = (struct osi_linux_mem *)data; |
| 180 | int memtype; |
| 181 | |
| 182 | memtype = MEMTYPE(lmp->chunk); |
| 183 | if (memtype != KM_TYPE && memtype != VM_TYPE) { |
| 184 | printf |
| 185 | ("osi_linux_verify_alloced_memory: unknown type %d at 0x%lx, index=%lu\n", |
| 186 | (int)memtype, (unsigned long)lmp->chunk, (unsigned long)index); |
| 187 | } |
| 188 | afs_linux_hash_verify_count++; |
| 189 | } |
| 190 | |
| 191 | |
| 192 | /* local_free() : wrapper for vfree(), to deal with incompatible protoypes */ |
| 193 | static void |
| 194 | local_free(void *p, size_t n) |
| 195 | { |
| 196 | vfree(p); |
| 197 | } |
| 198 | |
| 199 | /* linux_alloc_init(): Initializes the kernel memory allocator. As part |
| 200 | * of this process, it also initializes a pool of osi_linux_mem |
| 201 | * structures as well as the hash table itself. |
| 202 | * Return values: |
| 203 | * 0 - failure |
| 204 | * 1 - success |
| 205 | */ |
| 206 | static int |
| 207 | linux_alloc_init(void) |
| 208 | { |
| 209 | /* initiate our pool of osi_linux_mem structs */ |
| 210 | al_mem_pool = |
| 211 | afs_atomlist_create(sizeof(struct osi_linux_mem), sizeof(long) * 1024, |
| 212 | (void *)vmalloc, local_free); |
| 213 | if (!al_mem_pool) { |
| 214 | printf("afs_osi_Alloc: Error in initialization(atomlist_create)\n"); |
| 215 | return 0; |
| 216 | } |
| 217 | |
| 218 | /* initialize the hash table to hold references to alloc'ed chunks */ |
| 219 | lh_mem_htab = afs_lhash_create(hash_equal, (void *)vmalloc, local_free); |
| 220 | if (!lh_mem_htab) { |
| 221 | printf("afs_osi_Alloc: Error in initialization(lhash_create)\n"); |
| 222 | return 0; |
| 223 | } |
| 224 | |
| 225 | return 1; |
| 226 | |
| 227 | } |
| 228 | |
| 229 | /************** Linux memory allocator interface functions **********/ |
| 230 | |
| 231 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,16) |
| 232 | DEFINE_MUTEX(afs_linux_alloc_sem); |
| 233 | #else |
| 234 | DECLARE_MUTEX(afs_linux_alloc_sem); |
| 235 | #endif |
| 236 | |
| 237 | void * |
| 238 | osi_linux_alloc(unsigned int asize, int drop_glock) |
| 239 | { |
| 240 | void *new = NULL; |
| 241 | struct osi_linux_mem *lmem; |
| 242 | |
| 243 | new = linux_alloc(asize, drop_glock); /* get a chunk of memory of size asize */ |
| 244 | |
| 245 | if (!new) { |
| 246 | printf("afs_osi_Alloc: Can't vmalloc %d bytes.\n", asize); |
| 247 | return new; |
| 248 | } |
| 249 | |
| 250 | mutex_lock(&afs_linux_alloc_sem); |
| 251 | |
| 252 | /* allocator hasn't been initialized yet */ |
| 253 | if (allocator_init == 0) { |
| 254 | if (linux_alloc_init() == 0) { |
| 255 | goto error; |
| 256 | } |
| 257 | allocator_init = 1; /* initialization complete */ |
| 258 | } |
| 259 | |
| 260 | /* get an atom to store the pointer to the chunk */ |
| 261 | lmem = (struct osi_linux_mem *)afs_atomlist_get(al_mem_pool); |
| 262 | if (!lmem) { |
| 263 | printf("afs_osi_Alloc: atomlist_get() failed."); |
| 264 | goto free_error; |
| 265 | } |
| 266 | /* store the chunk reference */ |
| 267 | lmem->chunk = new; |
| 268 | |
| 269 | /* hash in the chunk */ |
| 270 | if (afs_lhash_enter(lh_mem_htab, hash_chunk(new), lmem) != 0) { |
| 271 | printf("afs_osi_Alloc: lhash_enter failed\n"); |
| 272 | goto free_error; |
| 273 | } |
| 274 | afs_linux_cur_allocs++; /* no. of current allocations */ |
| 275 | afs_linux_total_allocs++; /* total no. of allocations done so far */ |
| 276 | error: |
| 277 | mutex_unlock(&afs_linux_alloc_sem); |
| 278 | return MEMADDR(new); |
| 279 | |
| 280 | free_error: |
| 281 | if (new) { |
| 282 | linux_free(new); |
| 283 | } |
| 284 | new = NULL; |
| 285 | goto error; |
| 286 | |
| 287 | |
| 288 | } |
| 289 | |
| 290 | /* osi_linux_free() - free chunk of memory passed to us. |
| 291 | */ |
| 292 | void |
| 293 | osi_linux_free(void *addr) |
| 294 | { |
| 295 | struct osi_linux_mem lmem, *lmp; |
| 296 | |
| 297 | mutex_lock(&afs_linux_alloc_sem); |
| 298 | |
| 299 | lmem.chunk = addr; |
| 300 | /* remove this chunk from our hash table */ |
| 301 | if ((lmp = |
| 302 | (struct osi_linux_mem *)afs_lhash_remove(lh_mem_htab, |
| 303 | hash_chunk(addr), &lmem))) { |
| 304 | linux_free(lmp->chunk); /* this contains the piggybacked type info */ |
| 305 | afs_atomlist_put(al_mem_pool, lmp); /* return osi_linux_mem struct to pool */ |
| 306 | afs_linux_cur_allocs--; |
| 307 | } else { |
| 308 | printf("osi_linux_free: failed to remove chunk from hashtable\n"); |
| 309 | BUG(); |
| 310 | } |
| 311 | |
| 312 | mutex_unlock(&afs_linux_alloc_sem); |
| 313 | } |
| 314 | |
| 315 | /* osi_linux_free_afs_memory() - free all chunks of memory allocated. |
| 316 | */ |
| 317 | void |
| 318 | osi_linux_free_afs_memory(void) |
| 319 | { |
| 320 | mutex_lock(&afs_linux_alloc_sem); |
| 321 | |
| 322 | if (allocator_init) { |
| 323 | /* iterate through all elements in the hash table and free both |
| 324 | * the chunk and the atom associated with it. |
| 325 | */ |
| 326 | afs_lhash_iter(lh_mem_htab, hash_free); |
| 327 | |
| 328 | /* free the atomlist. */ |
| 329 | afs_atomlist_destroy(al_mem_pool); |
| 330 | |
| 331 | /* free the hashlist. */ |
| 332 | afs_lhash_destroy(lh_mem_htab); |
| 333 | |
| 334 | /* change the state so that the allocator is now uninitialized. */ |
| 335 | allocator_init = 0; |
| 336 | } |
| 337 | mutex_unlock(&afs_linux_alloc_sem); |
| 338 | } |
| 339 | |
| 340 | /* osi_linux_verify_alloced_memory(): verify all chunks of alloced memory in |
| 341 | * our hash table. |
| 342 | */ |
| 343 | void |
| 344 | osi_linux_verify_alloced_memory() |
| 345 | { |
| 346 | mutex_lock(&afs_linux_alloc_sem); |
| 347 | |
| 348 | /* count of times hash_verify was called. reset it to 0 before iteration */ |
| 349 | afs_linux_hash_verify_count = 0; |
| 350 | |
| 351 | /* iterate thru elements in the hash table */ |
| 352 | afs_lhash_iter(lh_mem_htab, hash_verify); |
| 353 | |
| 354 | if (afs_linux_hash_verify_count != afs_linux_cur_allocs) { |
| 355 | /* hmm, some pieces of memory are missing. */ |
| 356 | printf |
| 357 | ("osi_linux_verify_alloced_memory: %d chunks of memory are not accounted for during verify!\n", |
| 358 | afs_linux_hash_verify_count - afs_linux_cur_allocs); |
| 359 | } |
| 360 | |
| 361 | mutex_unlock(&afs_linux_alloc_sem); |
| 362 | return; |
| 363 | } |
| 364 | |
| 365 | #ifdef AFS_PRIVATE_OSI_ALLOCSPACES |
| 366 | |
| 367 | void |
| 368 | osi_FreeLargeSpace(void *p) |
| 369 | { |
| 370 | kfree(p); |
| 371 | } |
| 372 | |
| 373 | void |
| 374 | osi_FreeSmallSpace(void *p) |
| 375 | { |
| 376 | kfree(p); |
| 377 | } |
| 378 | |
| 379 | void * |
| 380 | osi_AllocLargeSpace(size_t size) |
| 381 | { |
| 382 | if (size > AFS_LRALLOCSIZ) |
| 383 | osi_Panic("osi_AllocLargeSpace: size=%d\n", (int) size); |
| 384 | return kmalloc(AFS_LRALLOCSIZ, GFP_NOFS); |
| 385 | } |
| 386 | |
| 387 | void * |
| 388 | osi_AllocSmallSpace(size_t size) |
| 389 | { |
| 390 | if (size > AFS_SMALLOCSIZ) |
| 391 | osi_Panic("osi_AllocSmallS: size=%d\n", (int)size); |
| 392 | return kmalloc(AFS_SMALLOCSIZ, GFP_NOFS); |
| 393 | } |
| 394 | #endif /* AFS_PRIVATE_OSI_ALLOCSPACES */ |