src/rx/rx_atomic.h

   1 /*
   2  * Copyright (c) 2010 Your Filesystem Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR `AS IS'' AND ANY EXPRESS OR
  14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #ifndef OPENAFS_RX_ATOMIC_H
  26 #define OPENAFS_RX_ATOMIC_H 1
  27
  28 #define RX_ATOMIC_INIT(i) { (i) }
  29
  30 #ifdef AFS_NT40_ENV
  31 typedef struct {
  32     volatile int var;
  33 } rx_atomic_t;
  34
  35 static_inline void
  36 rx_atomic_set(rx_atomic_t *atomic, int val) {
  37     atomic->var = val;
  38 }
  39
  40 static_inline int
  41 rx_atomic_read(rx_atomic_t *atomic) {
  42     return atomic->var;
  43 }
  44
  45 static_inline void
  46 rx_atomic_inc(rx_atomic_t *atomic) {
  47     InterlockedIncrement(&atomic->var);
  48 }
  49
  50 static_inline int
  51 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
  52     return InterlockedIncrement(&atomic->var);
  53 }
  54
  55 static_inline void
  56 rx_atomic_add(rx_atomic_t *atomic, int change) {
  57     InterlockedExchangeAdd(&atomic->var, change);
  58 }
  59
  60 static_inline int
  61 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
  62     return InterlockedExchangeAdd(&atomic->var, change) + change;
  63 }
  64
  65 static_inline void
  66 rx_atomic_dec(rx_atomic_t *atomic) {
  67     InterlockedDecrement(&atomic->var);
  68 }
  69
  70 static_inline int
  71 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
  72     return InterlockedDecrement(&atomic->var);
  73 }
  74
  75 static_inline void
  76 rx_atomic_sub(rx_atomic_t *atomic, int change) {
  77     InterlockedExchangeAdd(&atomic->var, 0 - change);
  78 }
  79
  80 static_inline int
  81 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
  82     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
  83 }
  84
  85 /* No InterlockedOr or InterlockedAnd on ix86, so just use the
  86  * BitTest functions */
  87
  88 static_inline void
  89 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
  90     (void) InterlockedBitTestAndSet(&atomic->var, bit);
  91 }
  92
  93 static_inline void
  94 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
  95     (void) InterlockedBitTestAndReset(&atomic->var, bit);
  96 }
  97
  98 static_inline int
  99 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 100     return InterlockedBitTestAndSet(&atomic->var, bit);
 101 }
 102
 103 static_inline int
 104 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 105     return InterlockedBitTestAndReset(&atomic->var, bit);
 106 }
 107
 108 #elif defined(AFS_AIX61_ENV) || defined(AFS_USR_AIX61_ENV)
 109 #include <sys/atomic_op.h>
 110
 111 typedef struct {
 112    volatile int var;
 113 } rx_atomic_t;
 114
 115 static_inline void
 116 rx_atomic_set(rx_atomic_t *atomic, int val) {
 117     atomic->var = val;
 118 }
 119
 120 static_inline int
 121 rx_atomic_read(rx_atomic_t *atomic) {
 122     return atomic->var;
 123 }
 124
 125 static_inline void
 126 rx_atomic_inc(rx_atomic_t *atomic) {
 127     fetch_and_add(&atomic->var, 1);
 128 }
 129
 130 static_inline int
 131 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
 132     return (fetch_and_add(&atomic->var, 1) + 1);
 133 }
 134
 135 static_inline void
 136 rx_atomic_add(rx_atomic_t *atomic, int change) {
 137     fetch_and_add(&atomic->var, change);
 138 }
 139
 140 static_inline int
 141 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
 142     return (fetch_and_add(&atomic->var, change) + change);
 143 }
 144
 145 static_inline void
 146 rx_atomic_dec(rx_atomic_t *atomic) {
 147     fetch_and_add(&atomic->var, -1);
 148 }
 149
 150 static_inline int
 151 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
 152     return (fetch_and_add(&atomic->var, -1) - 1);
 153 }
 154
 155 static_inline void
 156 rx_atomic_sub(rx_atomic_t *atomic, int change) {
 157     fetch_and_add(&atomic->var, -change);
 158 }
 159
 160 static_inline int
 161 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
 162     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
 163 }
 164
 165 static_inline void
 166 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
 167     fetch_and_or(&atomic->var, 1<<bit);
 168 }
 169
 170 static_inline void
 171 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
 172     fetch_and_and(&atomic->var, ~(1<<bit));
 173 }
 174
 175 static_inline int
 176 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 177     return (fetch_and_or(&atomic->var, (1<<bit)) & 1<<bit) != 0;
 178 }
 179
 180 static_inline int
 181 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 182     return (fetch_and_and(&atomic->var, ~(1<<bit)) & 1<<bit) != 0;
 183 }
 184
 185 #elif defined(AFS_DARWIN80_ENV) || defined(AFS_USR_DARWIN80_ENV)
 186
 187 # if (defined(AFS_DARWIN160_ENV) || defined(AFS_USR_DARWIN160_ENV)) && !defined(KERNEL)
 188 #  define OSATOMIC_USE_INLINED 1
 189 # endif
 190
 191 # include <libkern/OSAtomic.h>
 192
 193 # if defined(KERNEL) && !defined(UKERNEL)
 194 static_inline int
 195 OSAtomicIncrement32(volatile int *value)
 196 {
 197     return OSIncrementAtomic(value) + 1;
 198 }
 199
 200 static_inline int
 201 OSAtomicAdd32(int amount, volatile int *value)
 202 {
 203     return OSAddAtomic(amount, value) + amount;
 204 }
 205
 206 static_inline int
 207 OSAtomicDecrement32(volatile int *value)
 208 {
 209     return OSDecrementAtomic(value) - 1;
 210 }
 211
 212 static_inline unsigned int
 213 OSAtomicOr32(unsigned int mask, volatile unsigned int *value)
 214 {
 215     return OSBitOrAtomic(mask, value) | mask;
 216 }
 217
 218 static_inline unsigned int
 219 OSAtomicAnd32(unsigned int mask, volatile unsigned int *value)
 220 {
 221     return OSBitAndAtomic(mask, value) & mask;
 222 }
 223 #define OSAtomicOr32Orig  OSBitOrAtomic
 224 #define OSAtomicAnd32Orig OSBitAndAtomic
 225 # endif
 226
 227 typedef struct {
 228     volatile int var;
 229 } rx_atomic_t;
 230
 231 static_inline void
 232 rx_atomic_set(rx_atomic_t *atomic, int val) {
 233     atomic->var = val;
 234 }
 235
 236 static_inline int
 237 rx_atomic_read(rx_atomic_t *atomic) {
 238     return atomic->var;
 239 }
 240
 241 static_inline void
 242 rx_atomic_inc(rx_atomic_t *atomic) {
 243     OSAtomicIncrement32(&atomic->var);
 244 }
 245
 246 static_inline int
 247 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
 248     return OSAtomicIncrement32(&atomic->var);
 249 }
 250
 251 static_inline void
 252 rx_atomic_add(rx_atomic_t *atomic, int change) {
 253     OSAtomicAdd32(change, &atomic->var);
 254 }
 255
 256 static_inline int
 257 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
 258     return OSAtomicAdd32(change, &atomic->var);
 259 }
 260
 261 static_inline void
 262 rx_atomic_dec(rx_atomic_t *atomic) {
 263     OSAtomicDecrement32(&atomic->var);
 264 }
 265
 266 static_inline int
 267 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
 268     return OSAtomicDecrement32(&atomic->var);
 269 }
 270
 271 static_inline void
 272 rx_atomic_sub(rx_atomic_t *atomic, int change) {
 273     OSAtomicAdd32(0 - change, &atomic->var);
 274 }
 275
 276 static_inline int
 277 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
 278     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
 279 }
 280
 281 static_inline void
 282 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
 283     OSAtomicOr32(1<<bit, (volatile uint32_t *)&atomic->var);
 284 }
 285
 286 static_inline void
 287 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
 288     OSAtomicAnd32(~(1<<bit), (volatile uint32_t *)&atomic->var);
 289 }
 290
 291 static_inline int
 292 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 293     return ((OSAtomicOr32Orig(1<<bit, (volatile uint32_t *)&atomic->var) & 1<<bit)
 294                 != 0);
 295 }
 296
 297 static_inline int
 298 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 299     return ((OSAtomicAnd32Orig(~(1<<bit),
 300                                (volatile uint32_t *)&atomic->var) & 1<<bit)
 301             != 0);
 302 }
 303
 304 #elif defined(AFS_LINUX26_ENV) && defined(KERNEL)
 305 #include <asm/atomic.h>
 306
 307 typedef atomic_t rx_atomic_t;
 308
 309 #define rx_atomic_set(X, V)       atomic_set(X, V)
 310 #define rx_atomic_read(X)         atomic_read(X)
 311 #define rx_atomic_inc(X)          atomic_inc(X)
 312 #define rx_atomic_inc_and_read(X) atomic_inc_return(X)
 313 #define rx_atomic_add(X, V)       atomic_add(V, X)
 314 #define rx_atomic_add_and_read(X, V) atomic_add_return(V, X)
 315 #define rx_atomic_dec(X)          atomic_dec(X)
 316 #define rx_atomic_dec_and_read(X) atomic_dec_return(X)
 317 #define rx_atomic_sub(X, V)       atomic_sub(V, X)
 318 #define rx_atomic_test_bit(X, B)  test_bit(B, (unsigned long *) &(X)->counter)
 319 #define rx_atomic_set_bit(X, B)   set_bit(B, (unsigned long *) &(X)->counter)
 320 #define rx_atomic_clear_bit(X, B) clear_bit(B, (unsigned long *) &(X)->counter)
 321 #define rx_atomic_test_and_set_bit(X, B)    test_and_set_bit(B, (unsigned long *) &(X)->counter)
 322 #define rx_atomic_test_and_clear_bit(X, B)  test_and_clear_bit(B, (unsigned long *) &(X)->counter)
 323
 324 #elif defined(AFS_SUN510_ENV) || (defined(AFS_SUN5_ENV) && defined(KERNEL) && !defined(UKERNEL))
 325
 326 # if defined(KERNEL) && !defined(UKERNEL)
 327 #  include <sys/atomic.h>
 328 # else
 329 #  include <atomic.h>
 330 # endif
 331
 332 #ifndef AFS_SUN510_ENV
 333 # define atomic_inc_32(X)    atomic_add_32((X), 1)
 334 # define atomic_inc_32_nv(X) atomic_add_32_nv((X), 1)
 335 # define atomic_dec_32(X)    atomic_add_32((X), -1)
 336 # define atomic_dec_32_nv(X) atomic_add_32_nv((X), -1)
 337 #endif
 338
 339 typedef struct {
 340     volatile unsigned int var;
 341 } rx_atomic_t;
 342
 343 static_inline void
 344 rx_atomic_set(rx_atomic_t *atomic, int val) {
 345     atomic->var = val;
 346 }
 347
 348 static_inline int
 349 rx_atomic_read(rx_atomic_t *atomic) {
 350     return atomic->var;
 351 }
 352
 353 static_inline void
 354 rx_atomic_inc(rx_atomic_t *atomic) {
 355     atomic_inc_32(&atomic->var);
 356 }
 357
 358 static_inline int
 359 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
 360     return atomic_inc_32_nv(&atomic->var);
 361 }
 362
 363 static_inline void
 364 rx_atomic_add(rx_atomic_t *atomic, int change) {
 365     atomic_add_32(&atomic->var, change);
 366 }
 367
 368 static_inline int
 369 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
 370     return atomic_add_32_nv(&atomic->var, change);
 371 }
 372
 373 static_inline void
 374 rx_atomic_dec(rx_atomic_t *atomic) {
 375     atomic_dec_32(&atomic->var);
 376 }
 377
 378 static_inline int
 379 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
 380     return atomic_dec_32_nv(&atomic->var);
 381 }
 382
 383 static_inline void
 384 rx_atomic_sub(rx_atomic_t *atomic, int change) {
 385     atomic_add_32(&atomic->var, 0 - change);
 386 }
 387
 388 static_inline int
 389 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
 390     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
 391 }
 392
 393 static_inline void
 394 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
 395     atomic_or_32(&atomic->var, 1<<bit);
 396 }
 397
 398 static_inline void
 399 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
 400     atomic_and_32(&atomic->var, ~(1<<bit));
 401 }
 402
 403 static_inline int
 404 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 405     return (atomic_set_long_excl(&atomic->var, bit) == -1);
 406 }
 407
 408 static_inline int
 409 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 410     return (atomic_clear_long_excl(&atomic->var, bit) == 0);
 411 }
 412
 413 #elif defined(__GNUC__) && defined(HAVE_SYNC_FETCH_AND_ADD)
 414
 415 typedef struct {
 416    volatile int var;
 417 } rx_atomic_t;
 418
 419 static_inline void
 420 rx_atomic_set(rx_atomic_t *atomic, int val) {
 421     atomic->var = val;
 422 }
 423
 424 static_inline int
 425 rx_atomic_read(rx_atomic_t *atomic) {
 426     return atomic->var;
 427 }
 428
 429 static_inline void
 430 rx_atomic_inc(rx_atomic_t *atomic) {
 431     (void)__sync_fetch_and_add(&atomic->var, 1);
 432 }
 433
 434 static_inline int
 435 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
 436     return __sync_add_and_fetch(&atomic->var, 1);
 437 }
 438
 439 static_inline void
 440 rx_atomic_add(rx_atomic_t *atomic, int change) {
 441     (void)__sync_fetch_and_add(&atomic->var, change);
 442 }
 443
 444 static_inline int
 445 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
 446     return __sync_fetch_and_add(&atomic->var, change);
 447 }
 448
 449 static_inline void
 450 rx_atomic_dec(rx_atomic_t *atomic) {
 451     (void)__sync_fetch_and_sub(&atomic->var, 1);
 452 }
 453
 454 static_inline int
 455 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
 456     return __sync_sub_and_fetch(&atomic->var, 1);
 457 }
 458
 459 static_inline void
 460 rx_atomic_sub(rx_atomic_t *atomic, int change) {
 461     (void)__sync_fetch_and_sub(&atomic->var, change);
 462 }
 463
 464 static_inline int
 465 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
 466     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
 467 }
 468
 469 static_inline void
 470 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
 471     (void)__sync_fetch_and_or(&atomic->var, 1<<bit);
 472 }
 473
 474 static_inline void
 475 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
 476     (void)__sync_fetch_and_and(&atomic->var, ~(1<<bit));
 477 }
 478
 479 static_inline int
 480 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 481     return (__sync_fetch_and_or(&atomic->var, 1<<bit) & 1<<bit) != 0;
 482 }
 483
 484 static_inline int
 485 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 486     return (__sync_fetch_and_and(&atomic->var, ~(1<<bit)) & 1<<bit) != 0;
 487 }
 488
 489 #else
 490
 491 /* If we're on a platform where we have no idea how to do atomics,
 492  * then we fall back to using a single process wide mutex to protect
 493  * all atomic variables. This won't be the quickest thing ever.
 494  */
 495
 496 #ifdef RX_ENABLE_LOCKS
 497 extern afs_kmutex_t rx_atomic_mutex;
 498 #endif
 499
 500 typedef struct {
 501     int var;
 502 } rx_atomic_t;
 503
 504 static_inline void
 505 rx_atomic_set(rx_atomic_t *atomic, int val) {
 506     MUTEX_ENTER(&rx_atomic_mutex);
 507     atomic->var = val;
 508     MUTEX_EXIT(&rx_atomic_mutex);
 509 }
 510
 511 static_inline int
 512 rx_atomic_read(rx_atomic_t *atomic) {
 513     int out;
 514
 515     MUTEX_ENTER(&rx_atomic_mutex);
 516     out = atomic->var;
 517     MUTEX_EXIT(&rx_atomic_mutex);
 518
 519     return out;
 520 }
 521
 522 static_inline void
 523 rx_atomic_inc(rx_atomic_t *atomic) {
 524    MUTEX_ENTER(&rx_atomic_mutex);
 525    atomic->var++;
 526    MUTEX_EXIT(&rx_atomic_mutex);
 527 }
 528
 529 static_inline int
 530 rx_atomic_inc_and_read(rx_atomic_t *atomic) {
 531     int retval;
 532     MUTEX_ENTER(&rx_atomic_mutex);
 533     atomic->var++;
 534     retval = atomic->var;
 535     MUTEX_EXIT(&rx_atomic_mutex);
 536     return retval;
 537 }
 538
 539 static_inline void
 540 rx_atomic_add(rx_atomic_t *atomic, int change) {
 541     MUTEX_ENTER(&rx_atomic_mutex);
 542     atomic->var += change;
 543     MUTEX_EXIT(&rx_atomic_mutex);
 544 }
 545
 546 static_inline int
 547 rx_atomic_add_and_read(rx_atomic_t *atomic, int change) {
 548     int retval;
 549
 550     MUTEX_ENTER(&rx_atomic_mutex);
 551     atomic->var += change;
 552     retval = atomic->var;
 553     MUTEX_EXIT(&rx_atomic_mutex);
 554
 555     return retval;
 556 }
 557
 558 static_inline void
 559 rx_atomic_dec(rx_atomic_t *atomic) {
 560     MUTEX_ENTER(&rx_atomic_mutex);
 561     atomic->var--;
 562     MUTEX_EXIT(&rx_atomic_mutex);
 563 }
 564
 565 static_inline int
 566 rx_atomic_dec_and_read(rx_atomic_t *atomic) {
 567     int retval;
 568     MUTEX_ENTER(&rx_atomic_mutex);
 569     atomic->var--;
 570     retval = atomic->var;
 571     MUTEX_EXIT(&rx_atomic_mutex);
 572     return retval;
 573 }
 574
 575
 576 static_inline void
 577 rx_atomic_sub(rx_atomic_t *atomic, int change) {
 578     MUTEX_ENTER(&rx_atomic_mutex);
 579     atomic->var -= change;
 580     MUTEX_EXIT(&rx_atomic_mutex);
 581 }
 582
 583 static_inline int
 584 rx_atomic_test_bit(rx_atomic_t *atomic, int bit) {
 585     return ((unsigned int) rx_atomic_read(atomic) & 1<<bit) != 0;
 586 }
 587
 588 static_inline void
 589 rx_atomic_set_bit(rx_atomic_t *atomic, int bit) {
 590     MUTEX_ENTER(&rx_atomic_mutex);
 591     atomic->var |= (1<<bit);
 592     MUTEX_EXIT(&rx_atomic_mutex);
 593 }
 594
 595 static_inline void
 596 rx_atomic_clear_bit(rx_atomic_t *atomic, int bit) {
 597     MUTEX_ENTER(&rx_atomic_mutex);
 598     atomic->var &= ~(1<<bit);
 599     MUTEX_EXIT(&rx_atomic_mutex);
 600 }
 601
 602 static_inline int
 603 rx_atomic_test_and_set_bit(rx_atomic_t *atomic, int bit) {
 604     int val;
 605
 606     MUTEX_ENTER(&rx_atomic_mutex);
 607     val = atomic->var;
 608     atomic->var |= 1<<bit;
 609     MUTEX_EXIT(&rx_atomic_mutex);
 610
 611     return (val & 1<<bit) == 1<<bit;
 612 }
 613
 614 static_inline int
 615 rx_atomic_test_and_clear_bit(rx_atomic_t *atomic, int bit) {
 616     int val;
 617
 618     MUTEX_ENTER(&rx_atomic_mutex);
 619     val = atomic->var;
 620     atomic->var &= ~(1<<bit);
 621     MUTEX_EXIT(&rx_atomic_mutex);
 622
 623     return (val & 1<<bit) == 1<<bit;
 624 }
 625 #endif
 626
 627 #endif