1 /************************************************************** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, 14 * software distributed under the License is distributed on an 15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 * KIND, either express or implied. See the License for the 17 * specific language governing permissions and limitations 18 * under the License. 19 * 20 *************************************************************/ 21 22 23 24 #include "alloc_cache.h" 25 #include "alloc_impl.h" 26 #include "alloc_arena.h" 27 #include "internal/once.h" 28 #include "sal/macros.h" 29 #include "osl/diagnose.h" 30 31 #ifndef INCLUDED_STRING_H 32 #include <string.h> 33 #endif 34 35 #ifndef INCLUDED_STDIO_H 36 #include <stdio.h> 37 #endif 38 39 #ifdef OS2 40 #undef OSL_TRACE 41 #define OSL_TRACE 1 ? ((void)0) : _OSL_GLOBAL osl_trace 42 43 #define INCL_DOS 44 #include <os2.h> 45 46 #endif 47 48 /* ================================================================= * 49 * 50 * cache internals. 51 * 52 * ================================================================= */ 53 54 /** g_cache_list 55 * @internal 56 */ 57 struct rtl_cache_list_st 58 { 59 rtl_memory_lock_type m_lock; 60 rtl_cache_type m_cache_head; 61 62 #if defined(SAL_UNX) 63 pthread_t m_update_thread; 64 pthread_cond_t m_update_cond; 65 #elif defined(SAL_OS2) 66 TID m_update_thread; 67 HEV m_update_cond; 68 #elif defined(SAL_W32) 69 HANDLE m_update_thread; 70 HANDLE m_update_cond; 71 #endif /* SAL_UNX || SAL_W32 */ 72 int m_update_done; 73 }; 74 75 static struct rtl_cache_list_st g_cache_list; 76 77 78 /** gp_cache_arena 79 * provided for cache_type allocations, and hash_table resizing. 80 * 81 * @internal 82 */ 83 static rtl_arena_type * gp_cache_arena = NULL; 84 85 86 /** gp_cache_magazine_cache 87 * @internal 88 */ 89 static rtl_cache_type * gp_cache_magazine_cache = NULL; 90 91 92 /** gp_cache_slab_cache 93 * @internal 94 */ 95 static rtl_cache_type * gp_cache_slab_cache = NULL; 96 97 98 /** gp_cache_bufctl_cache 99 * @internal 100 */ 101 static rtl_cache_type * gp_cache_bufctl_cache = NULL; 102 103 104 /** rtl_cache_init() 105 * @internal 106 */ 107 static int 108 rtl_cache_init (void); 109 110 111 /* ================================================================= */ 112 113 /** RTL_CACHE_HASH_INDEX() 114 */ 115 #define RTL_CACHE_HASH_INDEX_IMPL(a, s, q, m) \ 116 ((((a) + ((a) >> (s)) + ((a) >> ((s) << 1))) >> (q)) & (m)) 117 118 #define RTL_CACHE_HASH_INDEX(cache, addr) \ 119 RTL_CACHE_HASH_INDEX_IMPL((addr), (cache)->m_hash_shift, (cache)->m_type_shift, ((cache)->m_hash_size - 1)) 120 121 122 /** rtl_cache_hash_rescale() 123 */ 124 static void 125 rtl_cache_hash_rescale ( 126 rtl_cache_type * cache, 127 sal_Size new_size 128 ) 129 { 130 rtl_cache_bufctl_type ** new_table; 131 sal_Size new_bytes; 132 133 new_bytes = new_size * sizeof(rtl_cache_bufctl_type*); 134 new_table = (rtl_cache_bufctl_type**)rtl_arena_alloc(gp_cache_arena, &new_bytes); 135 136 if (new_table != NULL) 137 { 138 rtl_cache_bufctl_type ** old_table; 139 sal_Size old_size, i; 140 141 memset (new_table, 0, new_bytes); 142 143 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 144 145 old_table = cache->m_hash_table; 146 old_size = cache->m_hash_size; 147 148 OSL_TRACE( 149 "rtl_cache_hash_rescale(\"%s\"): " 150 "nbuf: % " PRIu64 " (ave: %" PRIu64 "), frees: %" PRIu64 " " 151 "[old_size: %lu, new_size: %lu]", 152 cache->m_name, 153 cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free, 154 (cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free) >> cache->m_hash_shift, 155 cache->m_slab_stats.m_free, 156 old_size, new_size); 157 158 cache->m_hash_table = new_table; 159 cache->m_hash_size = new_size; 160 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 161 162 for (i = 0; i < old_size; i++) 163 { 164 rtl_cache_bufctl_type * curr = old_table[i]; 165 while (curr != NULL) 166 { 167 rtl_cache_bufctl_type * next = curr->m_next; 168 rtl_cache_bufctl_type ** head; 169 170 head = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, curr->m_addr)]); 171 curr->m_next = (*head); 172 (*head) = curr; 173 174 curr = next; 175 } 176 old_table[i] = NULL; 177 } 178 179 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 180 181 if (old_table != cache->m_hash_table_0) 182 { 183 sal_Size old_bytes = old_size * sizeof(rtl_cache_bufctl_type*); 184 rtl_arena_free (gp_cache_arena, old_table, old_bytes); 185 } 186 } 187 } 188 189 /** rtl_cache_hash_insert() 190 */ 191 static RTL_MEMORY_INLINE sal_uIntPtr 192 rtl_cache_hash_insert ( 193 rtl_cache_type * cache, 194 rtl_cache_bufctl_type * bufctl 195 ) 196 { 197 rtl_cache_bufctl_type ** ppHead; 198 199 ppHead = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, bufctl->m_addr)]); 200 201 bufctl->m_next = (*ppHead); 202 (*ppHead) = bufctl; 203 204 return (bufctl->m_addr); 205 } 206 207 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 208 #pragma inline(rtl_cache_hash_insert) 209 #endif /* __SUNPRO_C */ 210 211 212 /** rtl_cache_hash_remove() 213 */ 214 static rtl_cache_bufctl_type * 215 rtl_cache_hash_remove ( 216 rtl_cache_type * cache, 217 sal_uIntPtr addr 218 ) 219 { 220 rtl_cache_bufctl_type ** ppHead; 221 rtl_cache_bufctl_type * bufctl; 222 sal_Size lookups = 0; 223 224 ppHead = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, addr)]); 225 while ((bufctl = *ppHead) != NULL) 226 { 227 if (bufctl->m_addr == addr) 228 { 229 *ppHead = bufctl->m_next, bufctl->m_next = NULL; 230 break; 231 } 232 233 lookups += 1; 234 ppHead = &(bufctl->m_next); 235 } 236 237 OSL_ASSERT (bufctl != NULL); /* bad free */ 238 239 if (lookups > 1) 240 { 241 sal_Size nbuf = (sal_Size)(cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free); 242 if (nbuf > 4 * cache->m_hash_size) 243 { 244 if (!(cache->m_features & RTL_CACHE_FEATURE_RESCALE)) 245 { 246 sal_Size ave = nbuf >> cache->m_hash_shift; 247 sal_Size new_size = cache->m_hash_size << (highbit(ave) - 1); 248 249 cache->m_features |= RTL_CACHE_FEATURE_RESCALE; 250 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 251 rtl_cache_hash_rescale (cache, new_size); 252 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 253 cache->m_features &= ~RTL_CACHE_FEATURE_RESCALE; 254 } 255 } 256 } 257 258 return (bufctl); 259 } 260 261 /* ================================================================= */ 262 263 /** RTL_CACHE_SLAB() 264 */ 265 #define RTL_CACHE_SLAB(addr, size) \ 266 (((rtl_cache_slab_type*)(RTL_MEMORY_P2END((sal_uIntPtr)(addr), (size)))) - 1) 267 268 269 /** rtl_cache_slab_constructor() 270 */ 271 static int 272 rtl_cache_slab_constructor (void * obj, void * arg) 273 { 274 rtl_cache_slab_type * slab = (rtl_cache_slab_type*)(obj); 275 276 (void) arg; /* unused */ 277 278 QUEUE_START_NAMED(slab, slab_); 279 slab->m_ntypes = 0; 280 281 return (1); 282 } 283 284 285 /** rtl_cache_slab_destructor() 286 */ 287 static void 288 rtl_cache_slab_destructor (void * obj, void * arg) 289 { 290 #if OSL_DEBUG_LEVEL == 0 291 (void) obj; /* unused */ 292 #else /* OSL_DEBUG_LEVEL */ 293 rtl_cache_slab_type * slab = (rtl_cache_slab_type*)(obj); 294 295 /* assure removed from queue(s) */ 296 OSL_ASSERT(QUEUE_STARTED_NAMED(slab, slab_)); 297 298 /* assure no longer referenced */ 299 OSL_ASSERT(slab->m_ntypes == 0); 300 #endif /* OSL_DEBUG_LEVEL */ 301 302 (void) arg; /* unused */ 303 } 304 305 306 /** rtl_cache_slab_create() 307 * 308 * @precond cache->m_slab_lock released. 309 */ 310 static rtl_cache_slab_type * 311 rtl_cache_slab_create ( 312 rtl_cache_type * cache 313 ) 314 { 315 rtl_cache_slab_type * slab = NULL; 316 void * addr; 317 sal_Size size; 318 319 size = cache->m_slab_size; 320 addr = rtl_arena_alloc (cache->m_source, &size); 321 if (addr != NULL) 322 { 323 OSL_ASSERT(size >= cache->m_slab_size); 324 325 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 326 { 327 /* allocate slab struct from slab cache */ 328 OSL_ASSERT (cache != gp_cache_slab_cache); 329 slab = (rtl_cache_slab_type*)rtl_cache_alloc (gp_cache_slab_cache); 330 } 331 else 332 { 333 /* construct embedded slab struct */ 334 slab = RTL_CACHE_SLAB(addr, cache->m_slab_size); 335 (void) rtl_cache_slab_constructor (slab, 0); 336 } 337 if (slab != NULL) 338 { 339 slab->m_data = (sal_uIntPtr)(addr); 340 341 /* dynamic freelist initialization */ 342 slab->m_bp = slab->m_data; 343 slab->m_sp = NULL; 344 } 345 else 346 { 347 rtl_arena_free (cache->m_source, addr, size); 348 } 349 } 350 return (slab); 351 } 352 353 354 /** rtl_cache_slab_destroy() 355 * 356 * @precond cache->m_slab_lock released. 357 */ 358 static void 359 rtl_cache_slab_destroy ( 360 rtl_cache_type * cache, 361 rtl_cache_slab_type * slab 362 ) 363 { 364 void * addr = (void*)(slab->m_data); 365 sal_Size refcnt = slab->m_ntypes; slab->m_ntypes = 0; 366 367 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 368 { 369 /* cleanup bufctl(s) for free buffer(s) */ 370 sal_Size ntypes = (slab->m_bp - slab->m_data) / cache->m_type_size; 371 for (ntypes -= refcnt; slab->m_sp != NULL; ntypes--) 372 { 373 rtl_cache_bufctl_type * bufctl = slab->m_sp; 374 375 /* pop from freelist */ 376 slab->m_sp = bufctl->m_next, bufctl->m_next = NULL; 377 378 /* return bufctl struct to bufctl cache */ 379 rtl_cache_free (gp_cache_bufctl_cache, bufctl); 380 } 381 OSL_ASSERT(ntypes == 0); 382 383 /* return slab struct to slab cache */ 384 rtl_cache_free (gp_cache_slab_cache, slab); 385 } 386 else 387 { 388 /* destruct embedded slab struct */ 389 rtl_cache_slab_destructor (slab, 0); 390 } 391 392 if ((refcnt == 0) || (cache->m_features & RTL_CACHE_FEATURE_BULKDESTROY)) 393 { 394 /* free memory */ 395 rtl_arena_free (cache->m_source, addr, cache->m_slab_size); 396 } 397 } 398 399 400 /** rtl_cache_slab_populate() 401 * 402 * @precond cache->m_slab_lock acquired. 403 */ 404 static int 405 rtl_cache_slab_populate ( 406 rtl_cache_type * cache 407 ) 408 { 409 rtl_cache_slab_type * slab; 410 411 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 412 slab = rtl_cache_slab_create (cache); 413 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 414 if (slab != NULL) 415 { 416 /* update buffer start addr w/ current color */ 417 slab->m_bp += cache->m_ncolor; 418 419 /* update color for next slab */ 420 cache->m_ncolor += cache->m_type_align; 421 if (cache->m_ncolor > cache->m_ncolor_max) 422 cache->m_ncolor = 0; 423 424 /* update stats */ 425 cache->m_slab_stats.m_mem_total += cache->m_slab_size; 426 427 /* insert onto 'free' queue */ 428 QUEUE_INSERT_HEAD_NAMED(&(cache->m_free_head), slab, slab_); 429 } 430 return (slab != NULL); 431 } 432 433 /* ================================================================= */ 434 435 /** rtl_cache_slab_alloc() 436 * 437 * Allocate a buffer from slab layer; used by magazine layer. 438 */ 439 static void * 440 rtl_cache_slab_alloc ( 441 rtl_cache_type * cache 442 ) 443 { 444 void * addr = NULL; 445 rtl_cache_slab_type * head; 446 447 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 448 449 head = &(cache->m_free_head); 450 if ((head->m_slab_next != head) || rtl_cache_slab_populate (cache)) 451 { 452 rtl_cache_slab_type * slab; 453 rtl_cache_bufctl_type * bufctl; 454 455 slab = head->m_slab_next; 456 OSL_ASSERT(slab->m_ntypes < cache->m_ntypes); 457 458 if (slab->m_sp == NULL) 459 { 460 /* initialize bufctl w/ current 'slab->m_bp' */ 461 OSL_ASSERT (slab->m_bp < slab->m_data + cache->m_ntypes * cache->m_type_size + cache->m_ncolor_max); 462 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 463 { 464 /* allocate bufctl */ 465 OSL_ASSERT (cache != gp_cache_bufctl_cache); 466 bufctl = (rtl_cache_bufctl_type*)rtl_cache_alloc (gp_cache_bufctl_cache); 467 if (bufctl == NULL) 468 { 469 /* out of memory */ 470 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 471 return (0); 472 } 473 474 bufctl->m_addr = slab->m_bp; 475 bufctl->m_slab = (sal_uIntPtr)(slab); 476 } 477 else 478 { 479 /* embedded bufctl */ 480 bufctl = (rtl_cache_bufctl_type*)(slab->m_bp); 481 } 482 bufctl->m_next = NULL; 483 484 /* update 'slab->m_bp' to next free buffer */ 485 slab->m_bp += cache->m_type_size; 486 487 /* assign bufctl to freelist */ 488 slab->m_sp = bufctl; 489 } 490 491 /* pop front */ 492 bufctl = slab->m_sp; 493 slab->m_sp = bufctl->m_next; 494 495 /* increment usage, check for full slab */ 496 if ((slab->m_ntypes += 1) == cache->m_ntypes) 497 { 498 /* remove from 'free' queue */ 499 QUEUE_REMOVE_NAMED(slab, slab_); 500 501 /* insert onto 'used' queue (tail) */ 502 QUEUE_INSERT_TAIL_NAMED(&(cache->m_used_head), slab, slab_); 503 } 504 505 /* update stats */ 506 cache->m_slab_stats.m_alloc += 1; 507 cache->m_slab_stats.m_mem_alloc += cache->m_type_size; 508 509 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 510 addr = (void*)rtl_cache_hash_insert (cache, bufctl); 511 else 512 addr = bufctl; 513 514 /* DEBUG ONLY: mark allocated, undefined */ 515 OSL_DEBUG_ONLY(memset(addr, 0x77777777, cache->m_type_size)); 516 VALGRIND_MEMPOOL_ALLOC(cache, addr, cache->m_type_size); 517 } 518 519 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 520 return (addr); 521 } 522 523 524 /** rtl_cache_slab_free() 525 * 526 * Return a buffer to slab layer; used by magazine layer. 527 */ 528 static void 529 rtl_cache_slab_free ( 530 rtl_cache_type * cache, 531 void * addr 532 ) 533 { 534 rtl_cache_bufctl_type * bufctl; 535 rtl_cache_slab_type * slab; 536 537 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 538 539 /* DEBUG ONLY: mark unallocated, undefined */ 540 VALGRIND_MEMPOOL_FREE(cache, addr); 541 /* OSL_DEBUG_ONLY() */ VALGRIND_MAKE_MEM_UNDEFINED(addr, cache->m_type_size); 542 OSL_DEBUG_ONLY(memset(addr, 0x33333333, cache->m_type_size)); 543 544 /* determine slab from addr */ 545 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 546 { 547 bufctl = rtl_cache_hash_remove (cache, (sal_uIntPtr)(addr)); 548 slab = (bufctl != NULL) ? (rtl_cache_slab_type*)(bufctl->m_slab) : 0; 549 } 550 else 551 { 552 /* embedded slab struct */ 553 bufctl = (rtl_cache_bufctl_type*)(addr); 554 slab = RTL_CACHE_SLAB(addr, cache->m_slab_size); 555 } 556 557 if (slab != NULL) 558 { 559 /* check for full slab */ 560 if (slab->m_ntypes == cache->m_ntypes) 561 { 562 /* remove from 'used' queue */ 563 QUEUE_REMOVE_NAMED(slab, slab_); 564 565 /* insert onto 'free' queue (head) */ 566 QUEUE_INSERT_HEAD_NAMED(&(cache->m_free_head), slab, slab_); 567 } 568 569 /* push front */ 570 bufctl->m_next = slab->m_sp; 571 slab->m_sp = bufctl; 572 573 /* update stats */ 574 cache->m_slab_stats.m_free += 1; 575 cache->m_slab_stats.m_mem_alloc -= cache->m_type_size; 576 577 /* decrement usage, check for empty slab */ 578 if ((slab->m_ntypes -= 1) == 0) 579 { 580 /* remove from 'free' queue */ 581 QUEUE_REMOVE_NAMED(slab, slab_); 582 583 /* update stats */ 584 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 585 586 /* free 'empty' slab */ 587 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 588 rtl_cache_slab_destroy (cache, slab); 589 return; 590 } 591 } 592 593 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 594 } 595 596 /* ================================================================= */ 597 598 /** rtl_cache_magazine_constructor() 599 */ 600 static int 601 rtl_cache_magazine_constructor (void * obj, void * arg) 602 { 603 rtl_cache_magazine_type * mag = (rtl_cache_magazine_type*)(obj); 604 /* @@@ sal_Size size = (sal_Size)(arg); @@@ */ 605 606 (void) arg; /* unused */ 607 608 mag->m_mag_next = NULL; 609 mag->m_mag_size = RTL_CACHE_MAGAZINE_SIZE; 610 mag->m_mag_used = 0; 611 612 return (1); 613 } 614 615 616 /** rtl_cache_magazine_destructor() 617 */ 618 static void 619 rtl_cache_magazine_destructor (void * obj, void * arg) 620 { 621 #if OSL_DEBUG_LEVEL == 0 622 (void) obj; /* unused */ 623 #else /* OSL_DEBUG_LEVEL */ 624 rtl_cache_magazine_type * mag = (rtl_cache_magazine_type*)(obj); 625 626 /* assure removed from queue(s) */ 627 OSL_ASSERT(mag->m_mag_next == NULL); 628 629 /* assure no longer referenced */ 630 OSL_ASSERT(mag->m_mag_used == 0); 631 #endif /* OSL_DEBUG_LEVEL */ 632 633 (void) arg; /* unused */ 634 } 635 636 637 /** rtl_cache_magazine_clear() 638 */ 639 static void 640 rtl_cache_magazine_clear ( 641 rtl_cache_type * cache, 642 rtl_cache_magazine_type * mag 643 ) 644 { 645 for (; mag->m_mag_used > 0; --mag->m_mag_used) 646 { 647 void * obj = mag->m_objects[mag->m_mag_used - 1]; 648 mag->m_objects[mag->m_mag_used - 1] = NULL; 649 650 /* DEBUG ONLY: mark cached object allocated, undefined */ 651 VALGRIND_MEMPOOL_ALLOC(cache, obj, cache->m_type_size); 652 if (cache->m_destructor != 0) 653 { 654 /* DEBUG ONLY: keep constructed object defined */ 655 VALGRIND_MAKE_MEM_DEFINED(obj, cache->m_type_size); 656 657 /* destruct object */ 658 (cache->m_destructor)(obj, cache->m_userarg); 659 } 660 661 /* return buffer to slab layer */ 662 rtl_cache_slab_free (cache, obj); 663 } 664 } 665 666 /* ================================================================= */ 667 668 /** rtl_cache_depot_enqueue() 669 * 670 * @precond cache->m_depot_lock acquired. 671 */ 672 static RTL_MEMORY_INLINE void 673 rtl_cache_depot_enqueue ( 674 rtl_cache_depot_type * depot, 675 rtl_cache_magazine_type * mag 676 ) 677 { 678 /* enqueue empty magazine */ 679 mag->m_mag_next = depot->m_mag_next; 680 depot->m_mag_next = mag; 681 682 /* update depot stats */ 683 depot->m_mag_count++; 684 } 685 686 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 687 #pragma inline(rtl_cache_depot_enqueue) 688 #endif /* __SUNPRO_C */ 689 690 691 /** rtl_cache_depot_dequeue() 692 * 693 * @precond cache->m_depot_lock acquired. 694 */ 695 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 696 rtl_cache_depot_dequeue ( 697 rtl_cache_depot_type * depot 698 ) 699 { 700 rtl_cache_magazine_type * mag = NULL; 701 if (depot->m_mag_count > 0) 702 { 703 /* dequeue magazine */ 704 OSL_ASSERT(depot->m_mag_next != NULL); 705 706 mag = depot->m_mag_next; 707 depot->m_mag_next = mag->m_mag_next; 708 mag->m_mag_next = NULL; 709 710 /* update depot stats */ 711 depot->m_mag_count--; 712 depot->m_curr_min = SAL_MIN(depot->m_curr_min, depot->m_mag_count); 713 } 714 return (mag); 715 } 716 717 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 718 #pragma inline(rtl_cache_depot_dequeue) 719 #endif /* __SUNPRO_C */ 720 721 722 /** rtl_cache_depot_exchange_alloc() 723 * 724 * @precond cache->m_depot_lock acquired. 725 */ 726 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 727 rtl_cache_depot_exchange_alloc ( 728 rtl_cache_type * cache, 729 rtl_cache_magazine_type * empty 730 ) 731 { 732 rtl_cache_magazine_type * full; 733 734 OSL_ASSERT((empty == NULL) || (empty->m_mag_used == 0)); 735 736 /* dequeue full magazine */ 737 full = rtl_cache_depot_dequeue (&(cache->m_depot_full)); 738 if ((full != NULL) && (empty != NULL)) 739 { 740 /* enqueue empty magazine */ 741 rtl_cache_depot_enqueue (&(cache->m_depot_empty), empty); 742 } 743 744 OSL_ASSERT((full == NULL) || (full->m_mag_used > 0)); 745 746 return (full); 747 } 748 749 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 750 #pragma inline(rtl_cache_depot_exchange_alloc) 751 #endif /* __SUNPRO_C */ 752 753 754 /** rtl_cache_depot_exchange_free() 755 * 756 * @precond cache->m_depot_lock acquired. 757 */ 758 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 759 rtl_cache_depot_exchange_free ( 760 rtl_cache_type * cache, 761 rtl_cache_magazine_type * full 762 ) 763 { 764 rtl_cache_magazine_type * empty; 765 766 OSL_ASSERT((full == NULL) || (full->m_mag_used > 0)); 767 768 /* dequeue empty magazine */ 769 empty = rtl_cache_depot_dequeue (&(cache->m_depot_empty)); 770 if ((empty != NULL) && (full != NULL)) 771 { 772 /* enqueue full magazine */ 773 rtl_cache_depot_enqueue (&(cache->m_depot_full), full); 774 } 775 776 OSL_ASSERT((empty == NULL) || (empty->m_mag_used == 0)); 777 778 return (empty); 779 } 780 781 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 782 #pragma inline(rtl_cache_depot_exchange_free) 783 #endif /* __SUNPRO_C */ 784 785 786 /** rtl_cache_depot_populate() 787 * 788 * @precond cache->m_depot_lock acquired. 789 */ 790 static int 791 rtl_cache_depot_populate ( 792 rtl_cache_type * cache 793 ) 794 { 795 rtl_cache_magazine_type * empty = NULL; 796 797 if (cache->m_magazine_cache != 0) 798 { 799 /* allocate new empty magazine */ 800 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 801 empty = (rtl_cache_magazine_type*)rtl_cache_alloc (cache->m_magazine_cache); 802 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 803 if (empty != NULL) 804 { 805 /* enqueue (new) empty magazine */ 806 rtl_cache_depot_enqueue (&(cache->m_depot_empty), empty); 807 } 808 } 809 return (empty != NULL); 810 } 811 812 /* ================================================================= */ 813 814 /** rtl_cache_constructor() 815 */ 816 static int 817 rtl_cache_constructor (void * obj) 818 { 819 rtl_cache_type * cache = (rtl_cache_type*)(obj); 820 821 memset (cache, 0, sizeof(rtl_cache_type)); 822 823 /* linkage */ 824 QUEUE_START_NAMED(cache, cache_); 825 826 /* slab layer */ 827 (void)RTL_MEMORY_LOCK_INIT(&(cache->m_slab_lock)); 828 829 QUEUE_START_NAMED(&(cache->m_free_head), slab_); 830 QUEUE_START_NAMED(&(cache->m_used_head), slab_); 831 832 cache->m_hash_table = cache->m_hash_table_0; 833 cache->m_hash_size = RTL_CACHE_HASH_SIZE; 834 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 835 836 /* depot layer */ 837 (void)RTL_MEMORY_LOCK_INIT(&(cache->m_depot_lock)); 838 839 return (1); 840 } 841 842 /** rtl_cache_destructor() 843 */ 844 static void 845 rtl_cache_destructor (void * obj) 846 { 847 rtl_cache_type * cache = (rtl_cache_type*)(obj); 848 849 /* linkage */ 850 OSL_ASSERT(QUEUE_STARTED_NAMED(cache, cache_)); 851 852 /* slab layer */ 853 (void)RTL_MEMORY_LOCK_DESTROY(&(cache->m_slab_lock)); 854 855 OSL_ASSERT(QUEUE_STARTED_NAMED(&(cache->m_free_head), slab_)); 856 OSL_ASSERT(QUEUE_STARTED_NAMED(&(cache->m_used_head), slab_)); 857 858 OSL_ASSERT(cache->m_hash_table == cache->m_hash_table_0); 859 OSL_ASSERT(cache->m_hash_size == RTL_CACHE_HASH_SIZE); 860 OSL_ASSERT(cache->m_hash_shift == (sal_Size)(highbit(cache->m_hash_size) - 1)); 861 862 /* depot layer */ 863 (void)RTL_MEMORY_LOCK_DESTROY(&(cache->m_depot_lock)); 864 } 865 866 /* ================================================================= */ 867 868 /** rtl_cache_activate() 869 */ 870 static rtl_cache_type * 871 rtl_cache_activate ( 872 rtl_cache_type * cache, 873 const char * name, 874 size_t objsize, 875 size_t objalign, 876 int (SAL_CALL * constructor)(void * obj, void * userarg), 877 void (SAL_CALL * destructor) (void * obj, void * userarg), 878 void (SAL_CALL * reclaim) (void * userarg), 879 void * userarg, 880 rtl_arena_type * source, 881 int flags 882 ) 883 { 884 OSL_ASSERT(cache != NULL); 885 if (cache != NULL) 886 { 887 sal_Size slabsize; 888 889 snprintf (cache->m_name, sizeof(cache->m_name), "%s", name); 890 891 /* ensure minimum size (embedded bufctl linkage) */ 892 objsize = SAL_MAX(objsize, sizeof(rtl_cache_bufctl_type*)); 893 894 if (objalign == 0) 895 { 896 /* determine default alignment */ 897 if (objsize >= RTL_MEMORY_ALIGNMENT_8) 898 objalign = RTL_MEMORY_ALIGNMENT_8; 899 else 900 objalign = RTL_MEMORY_ALIGNMENT_4; 901 } 902 else 903 { 904 /* ensure minimum alignment */ 905 objalign = SAL_MAX(objalign, RTL_MEMORY_ALIGNMENT_4); 906 } 907 OSL_ASSERT(RTL_MEMORY_ISP2(objalign)); 908 909 cache->m_type_size = objsize = RTL_MEMORY_P2ROUNDUP(objsize, objalign); 910 cache->m_type_align = objalign; 911 cache->m_type_shift = highbit(cache->m_type_size) - 1; 912 913 cache->m_constructor = constructor; 914 cache->m_destructor = destructor; 915 cache->m_reclaim = reclaim; 916 cache->m_userarg = userarg; 917 918 /* slab layer */ 919 cache->m_source = source; 920 921 slabsize = source->m_quantum; /* minimum slab size */ 922 if (flags & RTL_CACHE_FLAG_QUANTUMCACHE) 923 { 924 /* next power of 2 above 3 * qcache_max */ 925 slabsize = SAL_MAX(slabsize, (1UL << highbit(3 * source->m_qcache_max))); 926 } 927 else 928 { 929 /* waste at most 1/8 of slab */ 930 slabsize = SAL_MAX(slabsize, cache->m_type_size * 8); 931 } 932 933 slabsize = RTL_MEMORY_P2ROUNDUP(slabsize, source->m_quantum); 934 if (!RTL_MEMORY_ISP2(slabsize)) 935 slabsize = 1UL << highbit(slabsize); 936 cache->m_slab_size = slabsize; 937 938 if (cache->m_slab_size > source->m_quantum) 939 { 940 OSL_ASSERT(gp_cache_slab_cache != NULL); 941 OSL_ASSERT(gp_cache_bufctl_cache != NULL); 942 943 cache->m_features |= RTL_CACHE_FEATURE_HASH; 944 cache->m_ntypes = cache->m_slab_size / cache->m_type_size; 945 cache->m_ncolor_max = cache->m_slab_size % cache->m_type_size; 946 } 947 else 948 { 949 /* embedded slab struct */ 950 cache->m_ntypes = (cache->m_slab_size - sizeof(rtl_cache_slab_type)) / cache->m_type_size; 951 cache->m_ncolor_max = (cache->m_slab_size - sizeof(rtl_cache_slab_type)) % cache->m_type_size; 952 } 953 954 OSL_ASSERT(cache->m_ntypes > 0); 955 cache->m_ncolor = 0; 956 957 if (flags & RTL_CACHE_FLAG_BULKDESTROY) 958 { 959 /* allow bulk slab delete upon cache deactivation */ 960 cache->m_features |= RTL_CACHE_FEATURE_BULKDESTROY; 961 } 962 963 /* magazine layer */ 964 if (!(flags & RTL_CACHE_FLAG_NOMAGAZINE)) 965 { 966 OSL_ASSERT(gp_cache_magazine_cache != NULL); 967 cache->m_magazine_cache = gp_cache_magazine_cache; 968 } 969 970 /* insert into cache list */ 971 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 972 QUEUE_INSERT_TAIL_NAMED(&(g_cache_list.m_cache_head), cache, cache_); 973 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 974 } 975 return (cache); 976 } 977 978 /** rtl_cache_deactivate() 979 */ 980 static void 981 rtl_cache_deactivate ( 982 rtl_cache_type * cache 983 ) 984 { 985 int active = 1; 986 987 /* remove from cache list */ 988 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 989 active = QUEUE_STARTED_NAMED(cache, cache_) == 0; 990 QUEUE_REMOVE_NAMED(cache, cache_); 991 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 992 993 OSL_PRECOND(active, "rtl_cache_deactivate(): orphaned cache."); 994 995 /* cleanup magazine layer */ 996 if (cache->m_magazine_cache != 0) 997 { 998 rtl_cache_type * mag_cache; 999 rtl_cache_magazine_type * mag; 1000 1001 /* prevent recursion */ 1002 mag_cache = cache->m_magazine_cache, cache->m_magazine_cache = 0; 1003 1004 /* cleanup cpu layer */ 1005 if ((mag = cache->m_cpu_curr) != NULL) 1006 { 1007 cache->m_cpu_curr = 0; 1008 rtl_cache_magazine_clear (cache, mag); 1009 rtl_cache_free (mag_cache, mag); 1010 } 1011 if ((mag = cache->m_cpu_prev) != NULL) 1012 { 1013 cache->m_cpu_prev = 0; 1014 rtl_cache_magazine_clear (cache, mag); 1015 rtl_cache_free (mag_cache, mag); 1016 } 1017 1018 /* cleanup depot layer */ 1019 while ((mag = rtl_cache_depot_dequeue(&(cache->m_depot_full))) != NULL) 1020 { 1021 rtl_cache_magazine_clear (cache, mag); 1022 rtl_cache_free (mag_cache, mag); 1023 } 1024 while ((mag = rtl_cache_depot_dequeue(&(cache->m_depot_empty))) != NULL) 1025 { 1026 rtl_cache_magazine_clear (cache, mag); 1027 rtl_cache_free (mag_cache, mag); 1028 } 1029 } 1030 1031 OSL_TRACE( 1032 "rtl_cache_deactivate(\"%s\"): " 1033 "[slab]: allocs: %"PRIu64", frees: %"PRIu64"; total: %lu, used: %lu; " 1034 "[cpu]: allocs: %"PRIu64", frees: %"PRIu64"; " 1035 "[total]: allocs: %"PRIu64", frees: %"PRIu64"", 1036 cache->m_name, 1037 cache->m_slab_stats.m_alloc, cache->m_slab_stats.m_free, 1038 cache->m_slab_stats.m_mem_total, cache->m_slab_stats.m_mem_alloc, 1039 cache->m_cpu_stats.m_alloc, cache->m_cpu_stats.m_free, 1040 cache->m_slab_stats.m_alloc + cache->m_cpu_stats.m_alloc, 1041 cache->m_slab_stats.m_free + cache->m_cpu_stats.m_free 1042 ); 1043 1044 /* cleanup slab layer */ 1045 if (cache->m_slab_stats.m_alloc > cache->m_slab_stats.m_free) 1046 { 1047 OSL_TRACE( 1048 "rtl_cache_deactivate(\"%s\"): " 1049 "cleaning up %"PRIu64" leaked buffer(s) [%lu bytes] [%lu total]", 1050 cache->m_name, 1051 cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free, 1052 cache->m_slab_stats.m_mem_alloc, cache->m_slab_stats.m_mem_total 1053 ); 1054 1055 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 1056 { 1057 /* cleanup bufctl(s) for leaking buffer(s) */ 1058 sal_Size i, n = cache->m_hash_size; 1059 for (i = 0; i < n; i++) 1060 { 1061 rtl_cache_bufctl_type * bufctl; 1062 while ((bufctl = cache->m_hash_table[i]) != NULL) 1063 { 1064 /* pop from hash table */ 1065 cache->m_hash_table[i] = bufctl->m_next, bufctl->m_next = NULL; 1066 1067 /* return to bufctl cache */ 1068 rtl_cache_free (gp_cache_bufctl_cache, bufctl); 1069 } 1070 } 1071 } 1072 { 1073 /* force cleanup of remaining slabs */ 1074 rtl_cache_slab_type *head, *slab; 1075 1076 head = &(cache->m_used_head); 1077 for (slab = head->m_slab_next; slab != head; slab = head->m_slab_next) 1078 { 1079 /* remove from 'used' queue */ 1080 QUEUE_REMOVE_NAMED(slab, slab_); 1081 1082 /* update stats */ 1083 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 1084 1085 /* free slab */ 1086 rtl_cache_slab_destroy (cache, slab); 1087 } 1088 1089 head = &(cache->m_free_head); 1090 for (slab = head->m_slab_next; slab != head; slab = head->m_slab_next) 1091 { 1092 /* remove from 'free' queue */ 1093 QUEUE_REMOVE_NAMED(slab, slab_); 1094 1095 /* update stats */ 1096 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 1097 1098 /* free slab */ 1099 rtl_cache_slab_destroy (cache, slab); 1100 } 1101 } 1102 } 1103 1104 if (cache->m_hash_table != cache->m_hash_table_0) 1105 { 1106 rtl_arena_free ( 1107 gp_cache_arena, 1108 cache->m_hash_table, 1109 cache->m_hash_size * sizeof(rtl_cache_bufctl_type*)); 1110 1111 cache->m_hash_table = cache->m_hash_table_0; 1112 cache->m_hash_size = RTL_CACHE_HASH_SIZE; 1113 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 1114 } 1115 } 1116 1117 /* ================================================================= * 1118 * 1119 * cache implementation. 1120 * 1121 * ================================================================= */ 1122 1123 /** rtl_cache_create() 1124 */ 1125 rtl_cache_type * 1126 SAL_CALL rtl_cache_create ( 1127 const char * name, 1128 sal_Size objsize, 1129 sal_Size objalign, 1130 int (SAL_CALL * constructor)(void * obj, void * userarg), 1131 void (SAL_CALL * destructor) (void * obj, void * userarg), 1132 void (SAL_CALL * reclaim) (void * userarg), 1133 void * userarg, 1134 rtl_arena_type * source, 1135 int flags 1136 ) SAL_THROW_EXTERN_C() 1137 { 1138 rtl_cache_type * result = 0; 1139 sal_Size size = sizeof(rtl_cache_type); 1140 1141 try_alloc: 1142 result = (rtl_cache_type*)rtl_arena_alloc (gp_cache_arena, &size); 1143 if (result != 0) 1144 { 1145 rtl_cache_type * cache = result; 1146 VALGRIND_CREATE_MEMPOOL(cache, 0, 0); 1147 (void) rtl_cache_constructor (cache); 1148 1149 if (!source) 1150 { 1151 /* use default arena */ 1152 OSL_ASSERT(gp_default_arena != 0); 1153 source = gp_default_arena; 1154 } 1155 1156 result = rtl_cache_activate ( 1157 cache, 1158 name, 1159 objsize, 1160 objalign, 1161 constructor, 1162 destructor, 1163 reclaim, 1164 userarg, 1165 source, 1166 flags 1167 ); 1168 1169 if (result == 0) 1170 { 1171 /* activation failed */ 1172 rtl_cache_deactivate (cache); 1173 rtl_cache_destructor (cache); 1174 VALGRIND_DESTROY_MEMPOOL(cache); 1175 rtl_arena_free (gp_cache_arena, cache, size); 1176 } 1177 } 1178 else if (gp_cache_arena == 0) 1179 { 1180 if (rtl_cache_init()) 1181 { 1182 /* try again */ 1183 goto try_alloc; 1184 } 1185 } 1186 return (result); 1187 } 1188 1189 /** rtl_cache_destroy() 1190 */ 1191 void SAL_CALL rtl_cache_destroy ( 1192 rtl_cache_type * cache 1193 ) SAL_THROW_EXTERN_C() 1194 { 1195 if (cache != 0) 1196 { 1197 rtl_cache_deactivate (cache); 1198 rtl_cache_destructor (cache); 1199 VALGRIND_DESTROY_MEMPOOL(cache); 1200 rtl_arena_free (gp_cache_arena, cache, sizeof(rtl_cache_type)); 1201 } 1202 } 1203 1204 /** rtl_cache_alloc() 1205 */ 1206 void * 1207 SAL_CALL rtl_cache_alloc ( 1208 rtl_cache_type * cache 1209 ) SAL_THROW_EXTERN_C() 1210 { 1211 void * obj = 0; 1212 1213 if (cache == 0) 1214 return (0); 1215 1216 if (cache->m_cpu_curr != 0) 1217 { 1218 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1219 1220 for (;;) 1221 { 1222 /* take object from magazine layer */ 1223 rtl_cache_magazine_type *curr, *prev, *temp; 1224 1225 curr = cache->m_cpu_curr; 1226 if ((curr != 0) && (curr->m_mag_used > 0)) 1227 { 1228 obj = curr->m_objects[--curr->m_mag_used]; 1229 #if defined(HAVE_VALGRIND_MEMCHECK_H) 1230 VALGRIND_MEMPOOL_ALLOC(cache, obj, cache->m_type_size); 1231 if (cache->m_constructor != 0) 1232 { 1233 /* keep constructed object defined */ 1234 VALGRIND_MAKE_MEM_DEFINED(obj, cache->m_type_size); 1235 } 1236 #endif /* HAVE_VALGRIND_MEMCHECK_H */ 1237 cache->m_cpu_stats.m_alloc += 1; 1238 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1239 1240 return (obj); 1241 } 1242 1243 prev = cache->m_cpu_prev; 1244 if ((prev != 0) && (prev->m_mag_used > 0)) 1245 { 1246 temp = cache->m_cpu_curr; 1247 cache->m_cpu_curr = cache->m_cpu_prev; 1248 cache->m_cpu_prev = temp; 1249 1250 continue; 1251 } 1252 1253 temp = rtl_cache_depot_exchange_alloc (cache, prev); 1254 if (temp != 0) 1255 { 1256 cache->m_cpu_prev = cache->m_cpu_curr; 1257 cache->m_cpu_curr = temp; 1258 1259 continue; 1260 } 1261 1262 /* no full magazine: fall through to slab layer */ 1263 break; 1264 } 1265 1266 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1267 } 1268 1269 /* alloc buffer from slab layer */ 1270 obj = rtl_cache_slab_alloc (cache); 1271 if ((obj != 0) && (cache->m_constructor != 0)) 1272 { 1273 /* construct object */ 1274 if (!((cache->m_constructor)(obj, cache->m_userarg))) 1275 { 1276 /* construction failure */ 1277 rtl_cache_slab_free (cache, obj), obj = 0; 1278 } 1279 } 1280 return (obj); 1281 } 1282 1283 /** rtl_cache_free() 1284 */ 1285 void 1286 SAL_CALL rtl_cache_free ( 1287 rtl_cache_type * cache, 1288 void * obj 1289 ) SAL_THROW_EXTERN_C() 1290 { 1291 if ((obj != 0) && (cache != 0)) 1292 { 1293 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1294 1295 for (;;) 1296 { 1297 /* return object to magazine layer */ 1298 rtl_cache_magazine_type *curr, *prev, *temp; 1299 1300 curr = cache->m_cpu_curr; 1301 if ((curr != 0) && (curr->m_mag_used < curr->m_mag_size)) 1302 { 1303 curr->m_objects[curr->m_mag_used++] = obj; 1304 #if defined(HAVE_VALGRIND_MEMCHECK_H) 1305 VALGRIND_MEMPOOL_FREE(cache, obj); 1306 #endif /* HAVE_VALGRIND_MEMCHECK_H */ 1307 cache->m_cpu_stats.m_free += 1; 1308 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1309 1310 return; 1311 } 1312 1313 prev = cache->m_cpu_prev; 1314 if ((prev != 0) && (prev->m_mag_used == 0)) 1315 { 1316 temp = cache->m_cpu_curr; 1317 cache->m_cpu_curr = cache->m_cpu_prev; 1318 cache->m_cpu_prev = temp; 1319 1320 continue; 1321 } 1322 1323 temp = rtl_cache_depot_exchange_free (cache, prev); 1324 if (temp != 0) 1325 { 1326 cache->m_cpu_prev = cache->m_cpu_curr; 1327 cache->m_cpu_curr = temp; 1328 1329 continue; 1330 } 1331 1332 if (rtl_cache_depot_populate(cache) != 0) 1333 { 1334 continue; 1335 } 1336 1337 /* no empty magazine: fall through to slab layer */ 1338 break; 1339 } 1340 1341 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1342 1343 /* no space for constructed object in magazine layer */ 1344 if (cache->m_destructor != 0) 1345 { 1346 /* destruct object */ 1347 (cache->m_destructor)(obj, cache->m_userarg); 1348 } 1349 1350 /* return buffer to slab layer */ 1351 rtl_cache_slab_free (cache, obj); 1352 } 1353 } 1354 1355 /* ================================================================= * 1356 * 1357 * cache wsupdate (machdep) internals. 1358 * 1359 * ================================================================= */ 1360 1361 /** rtl_cache_wsupdate_init() 1362 * 1363 * @precond g_cache_list.m_lock initialized 1364 */ 1365 static void 1366 rtl_cache_wsupdate_init (void); 1367 1368 1369 /** rtl_cache_wsupdate_wait() 1370 * 1371 * @precond g_cache_list.m_lock acquired 1372 */ 1373 static void 1374 rtl_cache_wsupdate_wait ( 1375 unsigned int seconds 1376 ); 1377 1378 /** rtl_cache_wsupdate_fini() 1379 * 1380 */ 1381 static void 1382 rtl_cache_wsupdate_fini (void); 1383 1384 /* ================================================================= */ 1385 1386 #if defined(SAL_UNX) 1387 1388 #include <sys/time.h> 1389 1390 static void * 1391 rtl_cache_wsupdate_all (void * arg); 1392 1393 static void 1394 rtl_cache_wsupdate_init (void) 1395 { 1396 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1397 g_cache_list.m_update_done = 0; 1398 (void) pthread_cond_init (&(g_cache_list.m_update_cond), NULL); 1399 if (pthread_create ( 1400 &(g_cache_list.m_update_thread), NULL, rtl_cache_wsupdate_all, (void*)(10)) != 0) 1401 { 1402 /* failure */ 1403 g_cache_list.m_update_thread = (pthread_t)(0); 1404 } 1405 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1406 } 1407 1408 static void 1409 rtl_cache_wsupdate_wait (unsigned int seconds) 1410 { 1411 if (seconds > 0) 1412 { 1413 struct timeval now; 1414 struct timespec wakeup; 1415 1416 gettimeofday(&now, 0); 1417 wakeup.tv_sec = now.tv_sec + (seconds); 1418 wakeup.tv_nsec = now.tv_usec * 1000; 1419 1420 (void) pthread_cond_timedwait ( 1421 &(g_cache_list.m_update_cond), 1422 &(g_cache_list.m_lock), 1423 &wakeup); 1424 } 1425 } 1426 1427 static void 1428 rtl_cache_wsupdate_fini (void) 1429 { 1430 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1431 g_cache_list.m_update_done = 1; 1432 pthread_cond_signal (&(g_cache_list.m_update_cond)); 1433 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1434 1435 if (g_cache_list.m_update_thread != (pthread_t)(0)) 1436 pthread_join (g_cache_list.m_update_thread, NULL); 1437 } 1438 1439 /* ================================================================= */ 1440 1441 #elif defined(SAL_OS2) 1442 1443 static void 1444 rtl_cache_wsupdate_all (void * arg); 1445 1446 static void rtl_cache_fini (void); 1447 1448 static void 1449 rtl_cache_wsupdate_init (void) 1450 { 1451 ULONG ulThreadId; 1452 APIRET rc; 1453 1454 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1455 g_cache_list.m_update_done = 0; 1456 1457 // we use atexit() because this allows CRT exit to process handler before 1458 // threads are killed. Otherwise with __attribute__(destructor) this 1459 // function is called when DosExit starts processing DLL destruction 1460 // which happens after ALL threads have been killed... 1461 atexit( rtl_cache_fini); 1462 1463 //g_cache_list.m_update_cond = CreateEvent (0, TRUE, FALSE, 0); 1464 /* Warp3 FP29 or Warp4 FP4 or better required */ 1465 rc = DosCreateEventSem( NULL, &g_cache_list.m_update_cond, 0x0800, 0); 1466 1467 g_cache_list.m_update_thread = (ULONG) _beginthread( rtl_cache_wsupdate_all, NULL, 1468 65*1024, (void*) 10); 1469 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1470 } 1471 1472 static void 1473 rtl_cache_wsupdate_wait (unsigned int seconds) 1474 { 1475 APIRET rc; 1476 if (seconds > 0) 1477 { 1478 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1479 rc = DosWaitEventSem(g_cache_list.m_update_cond, seconds*1000); 1480 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1481 } 1482 } 1483 1484 static void 1485 rtl_cache_wsupdate_fini (void) 1486 { 1487 APIRET rc; 1488 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1489 g_cache_list.m_update_done = 1; 1490 rc = DosPostEventSem(g_cache_list.m_update_cond); 1491 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1492 rc = DosWaitThread(&g_cache_list.m_update_thread, DCWW_WAIT); 1493 } 1494 1495 /* ================================================================= */ 1496 1497 #elif defined(SAL_W32) 1498 1499 static DWORD WINAPI 1500 rtl_cache_wsupdate_all (void * arg); 1501 1502 static void 1503 rtl_cache_wsupdate_init (void) 1504 { 1505 DWORD dwThreadId; 1506 1507 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1508 g_cache_list.m_update_done = 0; 1509 g_cache_list.m_update_cond = CreateEvent (0, TRUE, FALSE, 0); 1510 1511 g_cache_list.m_update_thread = 1512 CreateThread (NULL, 0, rtl_cache_wsupdate_all, (LPVOID)(10), 0, &dwThreadId); 1513 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1514 } 1515 1516 static void 1517 rtl_cache_wsupdate_wait (unsigned int seconds) 1518 { 1519 if (seconds > 0) 1520 { 1521 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1522 WaitForSingleObject (g_cache_list.m_update_cond, (DWORD)(seconds * 1000)); 1523 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1524 } 1525 } 1526 1527 static void 1528 rtl_cache_wsupdate_fini (void) 1529 { 1530 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1531 g_cache_list.m_update_done = 1; 1532 SetEvent (g_cache_list.m_update_cond); 1533 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1534 1535 WaitForSingleObject (g_cache_list.m_update_thread, INFINITE); 1536 } 1537 1538 #endif /* SAL_UNX || SAL_W32 */ 1539 1540 /* ================================================================= */ 1541 1542 /** rtl_cache_depot_wsupdate() 1543 * update depot stats and purge excess magazines. 1544 * 1545 * @precond cache->m_depot_lock acquired 1546 */ 1547 static void 1548 rtl_cache_depot_wsupdate ( 1549 rtl_cache_type * cache, 1550 rtl_cache_depot_type * depot 1551 ) 1552 { 1553 sal_Size npurge; 1554 1555 depot->m_prev_min = depot->m_curr_min; 1556 depot->m_curr_min = depot->m_mag_count; 1557 1558 npurge = SAL_MIN(depot->m_curr_min, depot->m_prev_min); 1559 for (; npurge > 0; npurge--) 1560 { 1561 rtl_cache_magazine_type * mag = rtl_cache_depot_dequeue (depot); 1562 if (mag != NULL) 1563 { 1564 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1565 rtl_cache_magazine_clear (cache, mag); 1566 rtl_cache_free (cache->m_magazine_cache, mag); 1567 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1568 } 1569 } 1570 } 1571 1572 /** rtl_cache_wsupdate() 1573 * 1574 * @precond cache->m_depot_lock released 1575 */ 1576 static void 1577 rtl_cache_wsupdate ( 1578 rtl_cache_type * cache 1579 ) 1580 { 1581 if (cache->m_magazine_cache != 0) 1582 { 1583 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1584 1585 OSL_TRACE( 1586 "rtl_cache_wsupdate(\"%s\") " 1587 "[depot: count, curr_min, prev_min] " 1588 "full: %lu, %lu, %lu; empty: %lu, %lu, %lu", 1589 cache->m_name, 1590 cache->m_depot_full.m_mag_count, 1591 cache->m_depot_full.m_curr_min, 1592 cache->m_depot_full.m_prev_min, 1593 cache->m_depot_empty.m_mag_count, 1594 cache->m_depot_empty.m_curr_min, 1595 cache->m_depot_empty.m_prev_min 1596 ); 1597 1598 rtl_cache_depot_wsupdate (cache, &(cache->m_depot_full)); 1599 rtl_cache_depot_wsupdate (cache, &(cache->m_depot_empty)); 1600 1601 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1602 } 1603 } 1604 1605 /** rtl_cache_wsupdate_all() 1606 * 1607 */ 1608 #if defined(SAL_UNX) 1609 static void * 1610 #elif defined(SAL_OS2) 1611 static void 1612 #elif defined(SAL_W32) 1613 static DWORD WINAPI 1614 #endif /* SAL_UNX || SAL_W32 */ 1615 rtl_cache_wsupdate_all (void * arg) 1616 { 1617 unsigned int seconds = (unsigned int)SAL_INT_CAST(sal_uIntPtr, arg); 1618 1619 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1620 while (!g_cache_list.m_update_done) 1621 { 1622 rtl_cache_wsupdate_wait (seconds); 1623 if (!g_cache_list.m_update_done) 1624 { 1625 rtl_cache_type * head, * cache; 1626 1627 head = &(g_cache_list.m_cache_head); 1628 for (cache = head->m_cache_next; 1629 cache != head; 1630 cache = cache->m_cache_next) 1631 { 1632 rtl_cache_wsupdate (cache); 1633 } 1634 } 1635 } 1636 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1637 1638 #if !defined(SAL_OS2) 1639 return (0); 1640 #endif 1641 } 1642 1643 /* ================================================================= * 1644 * 1645 * cache initialization. 1646 * 1647 * ================================================================= */ 1648 1649 static void 1650 rtl_cache_once_init (void) 1651 { 1652 { 1653 /* list of caches */ 1654 RTL_MEMORY_LOCK_INIT(&(g_cache_list.m_lock)); 1655 (void) rtl_cache_constructor (&(g_cache_list.m_cache_head)); 1656 } 1657 { 1658 /* cache: internal arena */ 1659 OSL_ASSERT(gp_cache_arena == NULL); 1660 1661 gp_cache_arena = rtl_arena_create ( 1662 "rtl_cache_internal_arena", 1663 64, /* quantum */ 1664 0, /* no quantum caching */ 1665 NULL, /* default source */ 1666 rtl_arena_alloc, 1667 rtl_arena_free, 1668 0 /* flags */ 1669 ); 1670 OSL_ASSERT(gp_cache_arena != NULL); 1671 1672 /* check 'gp_default_arena' initialization */ 1673 OSL_ASSERT(gp_default_arena != NULL); 1674 } 1675 { 1676 /* cache: magazine cache */ 1677 static rtl_cache_type g_cache_magazine_cache; 1678 1679 OSL_ASSERT(gp_cache_magazine_cache == NULL); 1680 VALGRIND_CREATE_MEMPOOL(&g_cache_magazine_cache, 0, 0); 1681 (void) rtl_cache_constructor (&g_cache_magazine_cache); 1682 1683 gp_cache_magazine_cache = rtl_cache_activate ( 1684 &g_cache_magazine_cache, 1685 "rtl_cache_magazine_cache", 1686 sizeof(rtl_cache_magazine_type), /* objsize */ 1687 0, /* objalign */ 1688 rtl_cache_magazine_constructor, 1689 rtl_cache_magazine_destructor, 1690 0, /* reclaim */ 1691 0, /* userarg: NYI */ 1692 gp_default_arena, /* source */ 1693 RTL_CACHE_FLAG_NOMAGAZINE /* during bootstrap; activated below */ 1694 ); 1695 OSL_ASSERT(gp_cache_magazine_cache != NULL); 1696 1697 /* activate magazine layer */ 1698 g_cache_magazine_cache.m_magazine_cache = gp_cache_magazine_cache; 1699 } 1700 { 1701 /* cache: slab (struct) cache */ 1702 static rtl_cache_type g_cache_slab_cache; 1703 1704 OSL_ASSERT(gp_cache_slab_cache == NULL); 1705 VALGRIND_CREATE_MEMPOOL(&g_cache_slab_cache, 0, 0); 1706 (void) rtl_cache_constructor (&g_cache_slab_cache); 1707 1708 gp_cache_slab_cache = rtl_cache_activate ( 1709 &g_cache_slab_cache, 1710 "rtl_cache_slab_cache", 1711 sizeof(rtl_cache_slab_type), /* objsize */ 1712 0, /* objalign */ 1713 rtl_cache_slab_constructor, 1714 rtl_cache_slab_destructor, 1715 0, /* reclaim */ 1716 0, /* userarg: none */ 1717 gp_default_arena, /* source */ 1718 0 /* flags: none */ 1719 ); 1720 OSL_ASSERT(gp_cache_slab_cache != NULL); 1721 } 1722 { 1723 /* cache: bufctl cache */ 1724 static rtl_cache_type g_cache_bufctl_cache; 1725 1726 OSL_ASSERT(gp_cache_bufctl_cache == NULL); 1727 VALGRIND_CREATE_MEMPOOL(&g_cache_bufctl_cache, 0, 0); 1728 (void) rtl_cache_constructor (&g_cache_bufctl_cache); 1729 1730 gp_cache_bufctl_cache = rtl_cache_activate ( 1731 &g_cache_bufctl_cache, 1732 "rtl_cache_bufctl_cache", 1733 sizeof(rtl_cache_bufctl_type), /* objsize */ 1734 0, /* objalign */ 1735 0, /* constructor */ 1736 0, /* destructor */ 1737 0, /* reclaim */ 1738 0, /* userarg */ 1739 gp_default_arena, /* source */ 1740 0 /* flags: none */ 1741 ); 1742 OSL_ASSERT(gp_cache_bufctl_cache != NULL); 1743 } 1744 1745 rtl_cache_wsupdate_init(); 1746 } 1747 1748 static int 1749 rtl_cache_init (void) 1750 { 1751 static sal_once_type g_once = SAL_ONCE_INIT; 1752 SAL_ONCE(&g_once, rtl_cache_once_init); 1753 return (gp_cache_arena != NULL); 1754 } 1755 1756 /* ================================================================= */ 1757 1758 /* 1759 Issue http://udk.openoffice.org/issues/show_bug.cgi?id=92388 1760 1761 Mac OS X does not seem to support "__cxa__atexit", thus leading 1762 to the situation that "__attribute__((destructor))__" functions 1763 (in particular "rtl_{memory|cache|arena}_fini") become called 1764 _before_ global C++ object d'tors. 1765 1766 Delegated the call to "rtl_cache_fini()" into a dummy C++ object, 1767 see alloc_fini.cxx . 1768 */ 1769 #if defined(__GNUC__) && !defined(MACOSX) && !defined(SAL_OS2) 1770 static void rtl_cache_fini (void) __attribute__((destructor)); 1771 #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) 1772 #pragma fini(rtl_cache_fini) 1773 static void rtl_cache_fini (void); 1774 #endif /* __GNUC__ || __SUNPRO_C */ 1775 1776 void 1777 rtl_cache_fini (void) 1778 { 1779 if (gp_cache_arena != NULL) 1780 { 1781 rtl_cache_type * cache, * head; 1782 1783 rtl_cache_wsupdate_fini(); 1784 1785 if (gp_cache_bufctl_cache != NULL) 1786 { 1787 cache = gp_cache_bufctl_cache, gp_cache_bufctl_cache = NULL; 1788 rtl_cache_deactivate (cache); 1789 rtl_cache_destructor (cache); 1790 VALGRIND_DESTROY_MEMPOOL(cache); 1791 } 1792 if (gp_cache_slab_cache != NULL) 1793 { 1794 cache = gp_cache_slab_cache, gp_cache_slab_cache = NULL; 1795 rtl_cache_deactivate (cache); 1796 rtl_cache_destructor (cache); 1797 VALGRIND_DESTROY_MEMPOOL(cache); 1798 } 1799 if (gp_cache_magazine_cache != NULL) 1800 { 1801 cache = gp_cache_magazine_cache, gp_cache_magazine_cache = NULL; 1802 rtl_cache_deactivate (cache); 1803 rtl_cache_destructor (cache); 1804 VALGRIND_DESTROY_MEMPOOL(cache); 1805 } 1806 if (gp_cache_arena != NULL) 1807 { 1808 rtl_arena_destroy (gp_cache_arena); 1809 gp_cache_arena = NULL; 1810 } 1811 1812 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1813 head = &(g_cache_list.m_cache_head); 1814 for (cache = head->m_cache_next; cache != head; cache = cache->m_cache_next) 1815 { 1816 OSL_TRACE( 1817 "rtl_cache_fini(\"%s\") " 1818 "[slab]: allocs: %"PRIu64", frees: %"PRIu64"; total: %lu, used: %lu; " 1819 "[cpu]: allocs: %"PRIu64", frees: %"PRIu64"; " 1820 "[total]: allocs: %"PRIu64", frees: %"PRIu64"", 1821 cache->m_name, 1822 cache->m_slab_stats.m_alloc, cache->m_slab_stats.m_free, 1823 cache->m_slab_stats.m_mem_total, cache->m_slab_stats.m_mem_alloc, 1824 cache->m_cpu_stats.m_alloc, cache->m_cpu_stats.m_free, 1825 cache->m_slab_stats.m_alloc + cache->m_cpu_stats.m_alloc, 1826 cache->m_slab_stats.m_free + cache->m_cpu_stats.m_free 1827 ); 1828 } 1829 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1830 } 1831 } 1832 1833 /* ================================================================= */ 1834