1 /************************************************************** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, 14 * software distributed under the License is distributed on an 15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 * KIND, either express or implied. See the License for the 17 * specific language governing permissions and limitations 18 * under the License. 19 * 20 *************************************************************/ 21 22 23 24 #include "alloc_cache.h" 25 #include "alloc_impl.h" 26 #include "alloc_arena.h" 27 #include "internal/once.h" 28 #include "sal/macros.h" 29 #include "osl/diagnose.h" 30 31 #ifndef INCLUDED_STRING_H 32 #include <string.h> 33 #endif 34 35 #ifndef INCLUDED_STDIO_H 36 #include <stdio.h> 37 #endif 38 39 #ifdef OS2 40 #undef OSL_TRACE 41 #define OSL_TRACE 1 ? ((void)0) : _OSL_GLOBAL osl_trace 42 43 #define INCL_DOS 44 #include <os2.h> 45 46 #endif 47 48 /* ================================================================= * 49 * 50 * cache internals. 51 * 52 * ================================================================= */ 53 54 /** g_cache_list 55 * @internal 56 */ 57 struct rtl_cache_list_st 58 { 59 rtl_memory_lock_type m_lock; 60 rtl_cache_type m_cache_head; 61 62 #if defined(SAL_UNX) 63 pthread_t m_update_thread; 64 pthread_cond_t m_update_cond; 65 #elif defined(SAL_OS2) 66 TID m_update_thread; 67 HEV m_update_cond; 68 #elif defined(SAL_W32) 69 HANDLE m_update_thread; 70 HANDLE m_update_cond; 71 #endif /* SAL_UNX || SAL_W32 */ 72 int m_update_done; 73 }; 74 75 static struct rtl_cache_list_st g_cache_list; 76 77 78 /** gp_cache_arena 79 * provided for cache_type allocations, and hash_table resizing. 80 * 81 * @internal 82 */ 83 static rtl_arena_type * gp_cache_arena = NULL; 84 85 86 /** gp_cache_magazine_cache 87 * @internal 88 */ 89 static rtl_cache_type * gp_cache_magazine_cache = NULL; 90 91 92 /** gp_cache_slab_cache 93 * @internal 94 */ 95 static rtl_cache_type * gp_cache_slab_cache = NULL; 96 97 98 /** gp_cache_bufctl_cache 99 * @internal 100 */ 101 static rtl_cache_type * gp_cache_bufctl_cache = NULL; 102 103 104 /** rtl_cache_init() 105 * @internal 106 */ 107 static int 108 rtl_cache_init (void); 109 110 111 /* ================================================================= */ 112 113 /** RTL_CACHE_HASH_INDEX() 114 */ 115 #define RTL_CACHE_HASH_INDEX_IMPL(a, s, q, m) \ 116 ((((a) + ((a) >> (s)) + ((a) >> ((s) << 1))) >> (q)) & (m)) 117 118 #define RTL_CACHE_HASH_INDEX(cache, addr) \ 119 RTL_CACHE_HASH_INDEX_IMPL((addr), (cache)->m_hash_shift, (cache)->m_type_shift, ((cache)->m_hash_size - 1)) 120 121 122 /** rtl_cache_hash_rescale() 123 */ 124 static void 125 rtl_cache_hash_rescale ( 126 rtl_cache_type * cache, 127 sal_Size new_size 128 ) 129 { 130 rtl_cache_bufctl_type ** new_table; 131 sal_Size new_bytes; 132 133 new_bytes = new_size * sizeof(rtl_cache_bufctl_type*); 134 new_table = (rtl_cache_bufctl_type**)rtl_arena_alloc(gp_cache_arena, &new_bytes); 135 136 if (new_table != NULL) 137 { 138 rtl_cache_bufctl_type ** old_table; 139 sal_Size old_size, i; 140 141 memset (new_table, 0, new_bytes); 142 143 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 144 145 old_table = cache->m_hash_table; 146 old_size = cache->m_hash_size; 147 148 OSL_TRACE( 149 "rtl_cache_hash_rescale(\"%s\"): " 150 "nbuf: % " PRIu64 " (ave: %" PRIu64 "), frees: %" PRIu64 " " 151 "[old_size: %lu, new_size: %lu]", 152 cache->m_name, 153 cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free, 154 (cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free) >> cache->m_hash_shift, 155 cache->m_slab_stats.m_free, 156 old_size, new_size); 157 158 cache->m_hash_table = new_table; 159 cache->m_hash_size = new_size; 160 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 161 162 for (i = 0; i < old_size; i++) 163 { 164 rtl_cache_bufctl_type * curr = old_table[i]; 165 while (curr != NULL) 166 { 167 rtl_cache_bufctl_type * next = curr->m_next; 168 rtl_cache_bufctl_type ** head; 169 170 head = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, curr->m_addr)]); 171 curr->m_next = (*head); 172 (*head) = curr; 173 174 curr = next; 175 } 176 old_table[i] = NULL; 177 } 178 179 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 180 181 if (old_table != cache->m_hash_table_0) 182 { 183 sal_Size old_bytes = old_size * sizeof(rtl_cache_bufctl_type*); 184 rtl_arena_free (gp_cache_arena, old_table, old_bytes); 185 } 186 } 187 } 188 189 /** rtl_cache_hash_insert() 190 */ 191 static RTL_MEMORY_INLINE sal_uIntPtr 192 rtl_cache_hash_insert ( 193 rtl_cache_type * cache, 194 rtl_cache_bufctl_type * bufctl 195 ) 196 { 197 rtl_cache_bufctl_type ** ppHead; 198 199 ppHead = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, bufctl->m_addr)]); 200 201 bufctl->m_next = (*ppHead); 202 (*ppHead) = bufctl; 203 204 return (bufctl->m_addr); 205 } 206 207 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 208 #pragma inline(rtl_cache_hash_insert) 209 #endif /* __SUNPRO_C */ 210 211 212 /** rtl_cache_hash_remove() 213 */ 214 static rtl_cache_bufctl_type * 215 rtl_cache_hash_remove ( 216 rtl_cache_type * cache, 217 sal_uIntPtr addr 218 ) 219 { 220 rtl_cache_bufctl_type ** ppHead; 221 rtl_cache_bufctl_type * bufctl; 222 sal_Size lookups = 0; 223 224 ppHead = &(cache->m_hash_table[RTL_CACHE_HASH_INDEX(cache, addr)]); 225 while ((bufctl = *ppHead) != NULL) 226 { 227 if (bufctl->m_addr == addr) 228 { 229 *ppHead = bufctl->m_next, bufctl->m_next = NULL; 230 break; 231 } 232 233 lookups += 1; 234 ppHead = &(bufctl->m_next); 235 } 236 237 OSL_ASSERT (bufctl != NULL); /* bad free */ 238 239 if (lookups > 1) 240 { 241 sal_Size nbuf = (sal_Size)(cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free); 242 if (nbuf > 4 * cache->m_hash_size) 243 { 244 if (!(cache->m_features & RTL_CACHE_FEATURE_RESCALE)) 245 { 246 sal_Size ave = nbuf >> cache->m_hash_shift; 247 sal_Size new_size = cache->m_hash_size << (highbit(ave) - 1); 248 249 cache->m_features |= RTL_CACHE_FEATURE_RESCALE; 250 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 251 rtl_cache_hash_rescale (cache, new_size); 252 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 253 cache->m_features &= ~RTL_CACHE_FEATURE_RESCALE; 254 } 255 } 256 } 257 258 return (bufctl); 259 } 260 261 /* ================================================================= */ 262 263 /** RTL_CACHE_SLAB() 264 */ 265 #define RTL_CACHE_SLAB(addr, size) \ 266 (((rtl_cache_slab_type*)(RTL_MEMORY_P2END((sal_uIntPtr)(addr), (size)))) - 1) 267 268 269 /** rtl_cache_slab_constructor() 270 */ 271 static int 272 rtl_cache_slab_constructor (void * obj, void * arg) 273 { 274 rtl_cache_slab_type * slab = (rtl_cache_slab_type*)(obj); 275 276 (void) arg; /* unused */ 277 278 QUEUE_START_NAMED(slab, slab_); 279 slab->m_ntypes = 0; 280 281 return (1); 282 } 283 284 285 /** rtl_cache_slab_destructor() 286 */ 287 static void 288 rtl_cache_slab_destructor (void * obj, void * arg) 289 { 290 #if OSL_DEBUG_LEVEL == 0 291 (void) obj; /* unused */ 292 #else /* OSL_DEBUG_LEVEL */ 293 rtl_cache_slab_type * slab = (rtl_cache_slab_type*)(obj); 294 295 /* assure removed from queue(s) */ 296 OSL_ASSERT(QUEUE_STARTED_NAMED(slab, slab_)); 297 298 /* assure no longer referenced */ 299 OSL_ASSERT(slab->m_ntypes == 0); 300 #endif /* OSL_DEBUG_LEVEL */ 301 302 (void) arg; /* unused */ 303 } 304 305 306 /** rtl_cache_slab_create() 307 * 308 * @precond cache->m_slab_lock released. 309 */ 310 static rtl_cache_slab_type * 311 rtl_cache_slab_create ( 312 rtl_cache_type * cache 313 ) 314 { 315 rtl_cache_slab_type * slab = NULL; 316 void * addr; 317 sal_Size size; 318 319 size = cache->m_slab_size; 320 addr = rtl_arena_alloc (cache->m_source, &size); 321 if (addr != NULL) 322 { 323 OSL_ASSERT(size >= cache->m_slab_size); 324 325 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 326 { 327 /* allocate slab struct from slab cache */ 328 OSL_ASSERT (cache != gp_cache_slab_cache); 329 slab = (rtl_cache_slab_type*)rtl_cache_alloc (gp_cache_slab_cache); 330 } 331 else 332 { 333 /* construct embedded slab struct */ 334 slab = RTL_CACHE_SLAB(addr, cache->m_slab_size); 335 (void) rtl_cache_slab_constructor (slab, 0); 336 } 337 if (slab != NULL) 338 { 339 slab->m_data = (sal_uIntPtr)(addr); 340 341 /* dynamic freelist initialization */ 342 slab->m_bp = slab->m_data; 343 slab->m_sp = NULL; 344 } 345 else 346 { 347 rtl_arena_free (cache->m_source, addr, size); 348 } 349 } 350 return (slab); 351 } 352 353 354 /** rtl_cache_slab_destroy() 355 * 356 * @precond cache->m_slab_lock released. 357 */ 358 static void 359 rtl_cache_slab_destroy ( 360 rtl_cache_type * cache, 361 rtl_cache_slab_type * slab 362 ) 363 { 364 void * addr = (void*)(slab->m_data); 365 sal_Size refcnt = slab->m_ntypes; slab->m_ntypes = 0; 366 367 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 368 { 369 /* cleanup bufctl(s) for free buffer(s) */ 370 sal_Size ntypes = (slab->m_bp - slab->m_data) / cache->m_type_size; 371 for (ntypes -= refcnt; slab->m_sp != NULL; ntypes--) 372 { 373 rtl_cache_bufctl_type * bufctl = slab->m_sp; 374 375 /* pop from freelist */ 376 slab->m_sp = bufctl->m_next, bufctl->m_next = NULL; 377 378 /* return bufctl struct to bufctl cache */ 379 rtl_cache_free (gp_cache_bufctl_cache, bufctl); 380 } 381 OSL_ASSERT(ntypes == 0); 382 383 /* return slab struct to slab cache */ 384 rtl_cache_free (gp_cache_slab_cache, slab); 385 } 386 else 387 { 388 /* destruct embedded slab struct */ 389 rtl_cache_slab_destructor (slab, 0); 390 } 391 392 if ((refcnt == 0) || (cache->m_features & RTL_CACHE_FEATURE_BULKDESTROY)) 393 { 394 /* free memory */ 395 rtl_arena_free (cache->m_source, addr, cache->m_slab_size); 396 } 397 } 398 399 400 /** rtl_cache_slab_populate() 401 * 402 * @precond cache->m_slab_lock acquired. 403 */ 404 static int 405 rtl_cache_slab_populate ( 406 rtl_cache_type * cache 407 ) 408 { 409 rtl_cache_slab_type * slab; 410 411 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 412 slab = rtl_cache_slab_create (cache); 413 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 414 if (slab != NULL) 415 { 416 /* update buffer start addr w/ current color */ 417 slab->m_bp += cache->m_ncolor; 418 419 /* update color for next slab */ 420 cache->m_ncolor += cache->m_type_align; 421 if (cache->m_ncolor > cache->m_ncolor_max) 422 cache->m_ncolor = 0; 423 424 /* update stats */ 425 cache->m_slab_stats.m_mem_total += cache->m_slab_size; 426 427 /* insert onto 'free' queue */ 428 QUEUE_INSERT_HEAD_NAMED(&(cache->m_free_head), slab, slab_); 429 } 430 return (slab != NULL); 431 } 432 433 /* ================================================================= */ 434 435 /** rtl_cache_slab_alloc() 436 * 437 * Allocate a buffer from slab layer; used by magazine layer. 438 */ 439 static void * 440 rtl_cache_slab_alloc ( 441 rtl_cache_type * cache 442 ) 443 { 444 void * addr = NULL; 445 rtl_cache_slab_type * head; 446 447 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 448 449 head = &(cache->m_free_head); 450 if ((head->m_slab_next != head) || rtl_cache_slab_populate (cache)) 451 { 452 rtl_cache_slab_type * slab; 453 rtl_cache_bufctl_type * bufctl; 454 455 slab = head->m_slab_next; 456 OSL_ASSERT(slab->m_ntypes < cache->m_ntypes); 457 458 if (slab->m_sp == NULL) 459 { 460 /* initialize bufctl w/ current 'slab->m_bp' */ 461 OSL_ASSERT (slab->m_bp < slab->m_data + cache->m_ntypes * cache->m_type_size + cache->m_ncolor_max); 462 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 463 { 464 /* allocate bufctl */ 465 OSL_ASSERT (cache != gp_cache_bufctl_cache); 466 bufctl = (rtl_cache_bufctl_type*)rtl_cache_alloc (gp_cache_bufctl_cache); 467 if (bufctl == NULL) 468 { 469 /* out of memory */ 470 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 471 return (0); 472 } 473 474 bufctl->m_addr = slab->m_bp; 475 bufctl->m_slab = (sal_uIntPtr)(slab); 476 } 477 else 478 { 479 /* embedded bufctl */ 480 bufctl = (rtl_cache_bufctl_type*)(slab->m_bp); 481 } 482 bufctl->m_next = NULL; 483 484 /* update 'slab->m_bp' to next free buffer */ 485 slab->m_bp += cache->m_type_size; 486 487 /* assign bufctl to freelist */ 488 slab->m_sp = bufctl; 489 } 490 491 /* pop front */ 492 bufctl = slab->m_sp; 493 slab->m_sp = bufctl->m_next; 494 495 /* increment usage, check for full slab */ 496 if ((slab->m_ntypes += 1) == cache->m_ntypes) 497 { 498 /* remove from 'free' queue */ 499 QUEUE_REMOVE_NAMED(slab, slab_); 500 501 /* insert onto 'used' queue (tail) */ 502 QUEUE_INSERT_TAIL_NAMED(&(cache->m_used_head), slab, slab_); 503 } 504 505 /* update stats */ 506 cache->m_slab_stats.m_alloc += 1; 507 cache->m_slab_stats.m_mem_alloc += cache->m_type_size; 508 509 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 510 addr = (void*)rtl_cache_hash_insert (cache, bufctl); 511 else 512 addr = bufctl; 513 514 /* DEBUG ONLY: mark allocated, undefined */ 515 OSL_DEBUG_ONLY(memset(addr, 0x77777777, cache->m_type_size)); 516 VALGRIND_MEMPOOL_ALLOC(cache, addr, cache->m_type_size); 517 } 518 519 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 520 return (addr); 521 } 522 523 524 /** rtl_cache_slab_free() 525 * 526 * Return a buffer to slab layer; used by magazine layer. 527 */ 528 static void 529 rtl_cache_slab_free ( 530 rtl_cache_type * cache, 531 void * addr 532 ) 533 { 534 rtl_cache_bufctl_type * bufctl; 535 rtl_cache_slab_type * slab; 536 537 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_slab_lock)); 538 539 /* DEBUG ONLY: mark unallocated, undefined */ 540 VALGRIND_MEMPOOL_FREE(cache, addr); 541 /* OSL_DEBUG_ONLY() */ VALGRIND_MAKE_MEM_UNDEFINED(addr, cache->m_type_size); 542 OSL_DEBUG_ONLY(memset(addr, 0x33333333, cache->m_type_size)); 543 544 /* determine slab from addr */ 545 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 546 { 547 bufctl = rtl_cache_hash_remove (cache, (sal_uIntPtr)(addr)); 548 slab = (bufctl != NULL) ? (rtl_cache_slab_type*)(bufctl->m_slab) : 0; 549 } 550 else 551 { 552 /* embedded slab struct */ 553 bufctl = (rtl_cache_bufctl_type*)(addr); 554 slab = RTL_CACHE_SLAB(addr, cache->m_slab_size); 555 } 556 557 if (slab != NULL) 558 { 559 /* check for full slab */ 560 if (slab->m_ntypes == cache->m_ntypes) 561 { 562 /* remove from 'used' queue */ 563 QUEUE_REMOVE_NAMED(slab, slab_); 564 565 /* insert onto 'free' queue (head) */ 566 QUEUE_INSERT_HEAD_NAMED(&(cache->m_free_head), slab, slab_); 567 } 568 569 /* push front */ 570 bufctl->m_next = slab->m_sp; 571 slab->m_sp = bufctl; 572 573 /* update stats */ 574 cache->m_slab_stats.m_free += 1; 575 cache->m_slab_stats.m_mem_alloc -= cache->m_type_size; 576 577 /* decrement usage, check for empty slab */ 578 if ((slab->m_ntypes -= 1) == 0) 579 { 580 /* remove from 'free' queue */ 581 QUEUE_REMOVE_NAMED(slab, slab_); 582 583 /* update stats */ 584 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 585 586 /* free 'empty' slab */ 587 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 588 rtl_cache_slab_destroy (cache, slab); 589 return; 590 } 591 } 592 593 RTL_MEMORY_LOCK_RELEASE(&(cache->m_slab_lock)); 594 } 595 596 /* ================================================================= */ 597 598 /** rtl_cache_magazine_constructor() 599 */ 600 static int 601 rtl_cache_magazine_constructor (void * obj, void * arg) 602 { 603 rtl_cache_magazine_type * mag = (rtl_cache_magazine_type*)(obj); 604 /* @@@ sal_Size size = (sal_Size)(arg); @@@ */ 605 606 (void) arg; /* unused */ 607 608 mag->m_mag_next = NULL; 609 mag->m_mag_size = RTL_CACHE_MAGAZINE_SIZE; 610 mag->m_mag_used = 0; 611 612 return (1); 613 } 614 615 616 /** rtl_cache_magazine_destructor() 617 */ 618 static void 619 rtl_cache_magazine_destructor (void * obj, void * arg) 620 { 621 #if OSL_DEBUG_LEVEL == 0 622 (void) obj; /* unused */ 623 #else /* OSL_DEBUG_LEVEL */ 624 rtl_cache_magazine_type * mag = (rtl_cache_magazine_type*)(obj); 625 626 /* assure removed from queue(s) */ 627 OSL_ASSERT(mag->m_mag_next == NULL); 628 629 /* assure no longer referenced */ 630 OSL_ASSERT(mag->m_mag_used == 0); 631 #endif /* OSL_DEBUG_LEVEL */ 632 633 (void) arg; /* unused */ 634 } 635 636 637 /** rtl_cache_magazine_clear() 638 */ 639 static void 640 rtl_cache_magazine_clear ( 641 rtl_cache_type * cache, 642 rtl_cache_magazine_type * mag 643 ) 644 { 645 for (; mag->m_mag_used > 0; --mag->m_mag_used) 646 { 647 void * obj = mag->m_objects[mag->m_mag_used - 1]; 648 mag->m_objects[mag->m_mag_used - 1] = NULL; 649 650 /* DEBUG ONLY: mark cached object allocated, undefined */ 651 VALGRIND_MEMPOOL_ALLOC(cache, obj, cache->m_type_size); 652 if (cache->m_destructor != 0) 653 { 654 /* DEBUG ONLY: keep constructed object defined */ 655 VALGRIND_MAKE_MEM_DEFINED(obj, cache->m_type_size); 656 657 /* destruct object */ 658 (cache->m_destructor)(obj, cache->m_userarg); 659 } 660 661 /* return buffer to slab layer */ 662 rtl_cache_slab_free (cache, obj); 663 } 664 } 665 666 /* ================================================================= */ 667 668 /** rtl_cache_depot_enqueue() 669 * 670 * @precond cache->m_depot_lock acquired. 671 */ 672 static RTL_MEMORY_INLINE void 673 rtl_cache_depot_enqueue ( 674 rtl_cache_depot_type * depot, 675 rtl_cache_magazine_type * mag 676 ) 677 { 678 /* enqueue empty magazine */ 679 mag->m_mag_next = depot->m_mag_next; 680 depot->m_mag_next = mag; 681 682 /* update depot stats */ 683 depot->m_mag_count++; 684 } 685 686 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 687 #pragma inline(rtl_cache_depot_enqueue) 688 #endif /* __SUNPRO_C */ 689 690 691 /** rtl_cache_depot_dequeue() 692 * 693 * @precond cache->m_depot_lock acquired. 694 */ 695 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 696 rtl_cache_depot_dequeue ( 697 rtl_cache_depot_type * depot 698 ) 699 { 700 rtl_cache_magazine_type * mag = NULL; 701 if (depot->m_mag_count > 0) 702 { 703 /* dequeue magazine */ 704 OSL_ASSERT(depot->m_mag_next != NULL); 705 706 mag = depot->m_mag_next; 707 depot->m_mag_next = mag->m_mag_next; 708 mag->m_mag_next = NULL; 709 710 /* update depot stats */ 711 depot->m_mag_count--; 712 depot->m_curr_min = SAL_MIN(depot->m_curr_min, depot->m_mag_count); 713 } 714 return (mag); 715 } 716 717 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 718 #pragma inline(rtl_cache_depot_dequeue) 719 #endif /* __SUNPRO_C */ 720 721 722 /** rtl_cache_depot_exchange_alloc() 723 * 724 * @precond cache->m_depot_lock acquired. 725 */ 726 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 727 rtl_cache_depot_exchange_alloc ( 728 rtl_cache_type * cache, 729 rtl_cache_magazine_type * empty 730 ) 731 { 732 rtl_cache_magazine_type * full; 733 734 OSL_ASSERT((empty == NULL) || (empty->m_mag_used == 0)); 735 736 /* dequeue full magazine */ 737 full = rtl_cache_depot_dequeue (&(cache->m_depot_full)); 738 if ((full != NULL) && (empty != NULL)) 739 { 740 /* enqueue empty magazine */ 741 rtl_cache_depot_enqueue (&(cache->m_depot_empty), empty); 742 } 743 744 OSL_ASSERT((full == NULL) || (full->m_mag_used > 0)); 745 746 return (full); 747 } 748 749 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 750 #pragma inline(rtl_cache_depot_exchange_alloc) 751 #endif /* __SUNPRO_C */ 752 753 754 /** rtl_cache_depot_exchange_free() 755 * 756 * @precond cache->m_depot_lock acquired. 757 */ 758 static RTL_MEMORY_INLINE rtl_cache_magazine_type * 759 rtl_cache_depot_exchange_free ( 760 rtl_cache_type * cache, 761 rtl_cache_magazine_type * full 762 ) 763 { 764 rtl_cache_magazine_type * empty; 765 766 OSL_ASSERT((full == NULL) || (full->m_mag_used > 0)); 767 768 /* dequeue empty magazine */ 769 empty = rtl_cache_depot_dequeue (&(cache->m_depot_empty)); 770 if ((empty != NULL) && (full != NULL)) 771 { 772 /* enqueue full magazine */ 773 rtl_cache_depot_enqueue (&(cache->m_depot_full), full); 774 } 775 776 OSL_ASSERT((empty == NULL) || (empty->m_mag_used == 0)); 777 778 return (empty); 779 } 780 781 #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) 782 #pragma inline(rtl_cache_depot_exchange_free) 783 #endif /* __SUNPRO_C */ 784 785 786 /** rtl_cache_depot_populate() 787 * 788 * @precond cache->m_depot_lock acquired. 789 */ 790 static int 791 rtl_cache_depot_populate ( 792 rtl_cache_type * cache 793 ) 794 { 795 rtl_cache_magazine_type * empty = NULL; 796 797 if (cache->m_magazine_cache != 0) 798 { 799 /* allocate new empty magazine */ 800 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 801 empty = (rtl_cache_magazine_type*)rtl_cache_alloc (cache->m_magazine_cache); 802 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 803 if (empty != NULL) 804 { 805 /* enqueue (new) empty magazine */ 806 rtl_cache_depot_enqueue (&(cache->m_depot_empty), empty); 807 } 808 } 809 return (empty != NULL); 810 } 811 812 /* ================================================================= */ 813 814 /** rtl_cache_constructor() 815 */ 816 static int 817 rtl_cache_constructor (void * obj) 818 { 819 rtl_cache_type * cache = (rtl_cache_type*)(obj); 820 821 memset (cache, 0, sizeof(rtl_cache_type)); 822 823 /* linkage */ 824 QUEUE_START_NAMED(cache, cache_); 825 826 /* slab layer */ 827 (void)RTL_MEMORY_LOCK_INIT(&(cache->m_slab_lock)); 828 829 QUEUE_START_NAMED(&(cache->m_free_head), slab_); 830 QUEUE_START_NAMED(&(cache->m_used_head), slab_); 831 832 cache->m_hash_table = cache->m_hash_table_0; 833 cache->m_hash_size = RTL_CACHE_HASH_SIZE; 834 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 835 836 /* depot layer */ 837 (void)RTL_MEMORY_LOCK_INIT(&(cache->m_depot_lock)); 838 839 return (1); 840 } 841 842 /** rtl_cache_destructor() 843 */ 844 static void 845 rtl_cache_destructor (void * obj) 846 { 847 rtl_cache_type * cache = (rtl_cache_type*)(obj); 848 849 /* linkage */ 850 OSL_ASSERT(QUEUE_STARTED_NAMED(cache, cache_)); 851 852 /* slab layer */ 853 (void)RTL_MEMORY_LOCK_DESTROY(&(cache->m_slab_lock)); 854 855 OSL_ASSERT(QUEUE_STARTED_NAMED(&(cache->m_free_head), slab_)); 856 OSL_ASSERT(QUEUE_STARTED_NAMED(&(cache->m_used_head), slab_)); 857 858 OSL_ASSERT(cache->m_hash_table == cache->m_hash_table_0); 859 OSL_ASSERT(cache->m_hash_size == RTL_CACHE_HASH_SIZE); 860 OSL_ASSERT(cache->m_hash_shift == (sal_Size)(highbit(cache->m_hash_size) - 1)); 861 862 /* depot layer */ 863 (void)RTL_MEMORY_LOCK_DESTROY(&(cache->m_depot_lock)); 864 } 865 866 /* ================================================================= */ 867 868 /** rtl_cache_activate() 869 */ 870 static rtl_cache_type * 871 rtl_cache_activate ( 872 rtl_cache_type * cache, 873 const char * name, 874 size_t objsize, 875 size_t objalign, 876 int (SAL_CALL * constructor)(void * obj, void * userarg), 877 void (SAL_CALL * destructor) (void * obj, void * userarg), 878 void (SAL_CALL * reclaim) (void * userarg), 879 void * userarg, 880 rtl_arena_type * source, 881 int flags 882 ) 883 { 884 OSL_ASSERT(cache != NULL); 885 if (cache != NULL) 886 { 887 sal_Size slabsize; 888 889 snprintf (cache->m_name, sizeof(cache->m_name), "%s", name); 890 891 /* ensure minimum size (embedded bufctl linkage) */ 892 objsize = SAL_MAX(objsize, sizeof(rtl_cache_bufctl_type*)); 893 894 if (objalign == 0) 895 { 896 /* determine default alignment */ 897 #ifdef NEED_ALIGN16 898 if (objsize >= RTL_MEMORY_ALIGNMENT_16) 899 objalign = RTL_MEMORY_ALIGNMENT_16; 900 else if (objsize >= RTL_MEMORY_ALIGNMENT_8) 901 #else 902 if (objsize >= RTL_MEMORY_ALIGNMENT_8) 903 #endif 904 objalign = RTL_MEMORY_ALIGNMENT_8; 905 else 906 objalign = RTL_MEMORY_ALIGNMENT_4; 907 } 908 else 909 { 910 /* ensure minimum alignment */ 911 objalign = SAL_MAX(objalign, RTL_MEMORY_ALIGNMENT_4); 912 } 913 OSL_ASSERT(RTL_MEMORY_ISP2(objalign)); 914 915 cache->m_type_size = objsize = RTL_MEMORY_P2ROUNDUP(objsize, objalign); 916 cache->m_type_align = objalign; 917 cache->m_type_shift = highbit(cache->m_type_size) - 1; 918 919 cache->m_constructor = constructor; 920 cache->m_destructor = destructor; 921 cache->m_reclaim = reclaim; 922 cache->m_userarg = userarg; 923 924 /* slab layer */ 925 cache->m_source = source; 926 927 slabsize = source->m_quantum; /* minimum slab size */ 928 if (flags & RTL_CACHE_FLAG_QUANTUMCACHE) 929 { 930 /* next power of 2 above 3 * qcache_max */ 931 slabsize = SAL_MAX(slabsize, (1UL << highbit(3 * source->m_qcache_max))); 932 } 933 else 934 { 935 /* waste at most 1/8 of slab */ 936 slabsize = SAL_MAX(slabsize, cache->m_type_size * 8); 937 } 938 939 slabsize = RTL_MEMORY_P2ROUNDUP(slabsize, source->m_quantum); 940 if (!RTL_MEMORY_ISP2(slabsize)) 941 slabsize = 1UL << highbit(slabsize); 942 cache->m_slab_size = slabsize; 943 944 if (cache->m_slab_size > source->m_quantum) 945 { 946 OSL_ASSERT(gp_cache_slab_cache != NULL); 947 OSL_ASSERT(gp_cache_bufctl_cache != NULL); 948 949 cache->m_features |= RTL_CACHE_FEATURE_HASH; 950 cache->m_ntypes = cache->m_slab_size / cache->m_type_size; 951 cache->m_ncolor_max = cache->m_slab_size % cache->m_type_size; 952 } 953 else 954 { 955 /* embedded slab struct */ 956 cache->m_ntypes = (cache->m_slab_size - sizeof(rtl_cache_slab_type)) / cache->m_type_size; 957 cache->m_ncolor_max = (cache->m_slab_size - sizeof(rtl_cache_slab_type)) % cache->m_type_size; 958 } 959 960 OSL_ASSERT(cache->m_ntypes > 0); 961 cache->m_ncolor = 0; 962 963 if (flags & RTL_CACHE_FLAG_BULKDESTROY) 964 { 965 /* allow bulk slab delete upon cache deactivation */ 966 cache->m_features |= RTL_CACHE_FEATURE_BULKDESTROY; 967 } 968 969 /* magazine layer */ 970 if (!(flags & RTL_CACHE_FLAG_NOMAGAZINE)) 971 { 972 OSL_ASSERT(gp_cache_magazine_cache != NULL); 973 cache->m_magazine_cache = gp_cache_magazine_cache; 974 } 975 976 /* insert into cache list */ 977 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 978 QUEUE_INSERT_TAIL_NAMED(&(g_cache_list.m_cache_head), cache, cache_); 979 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 980 } 981 return (cache); 982 } 983 984 /** rtl_cache_deactivate() 985 */ 986 static void 987 rtl_cache_deactivate ( 988 rtl_cache_type * cache 989 ) 990 { 991 int active = 1; 992 993 /* remove from cache list */ 994 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 995 active = QUEUE_STARTED_NAMED(cache, cache_) == 0; 996 QUEUE_REMOVE_NAMED(cache, cache_); 997 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 998 999 OSL_PRECOND(active, "rtl_cache_deactivate(): orphaned cache."); 1000 1001 /* cleanup magazine layer */ 1002 if (cache->m_magazine_cache != 0) 1003 { 1004 rtl_cache_type * mag_cache; 1005 rtl_cache_magazine_type * mag; 1006 1007 /* prevent recursion */ 1008 mag_cache = cache->m_magazine_cache, cache->m_magazine_cache = 0; 1009 1010 /* cleanup cpu layer */ 1011 if ((mag = cache->m_cpu_curr) != NULL) 1012 { 1013 cache->m_cpu_curr = 0; 1014 rtl_cache_magazine_clear (cache, mag); 1015 rtl_cache_free (mag_cache, mag); 1016 } 1017 if ((mag = cache->m_cpu_prev) != NULL) 1018 { 1019 cache->m_cpu_prev = 0; 1020 rtl_cache_magazine_clear (cache, mag); 1021 rtl_cache_free (mag_cache, mag); 1022 } 1023 1024 /* cleanup depot layer */ 1025 while ((mag = rtl_cache_depot_dequeue(&(cache->m_depot_full))) != NULL) 1026 { 1027 rtl_cache_magazine_clear (cache, mag); 1028 rtl_cache_free (mag_cache, mag); 1029 } 1030 while ((mag = rtl_cache_depot_dequeue(&(cache->m_depot_empty))) != NULL) 1031 { 1032 rtl_cache_magazine_clear (cache, mag); 1033 rtl_cache_free (mag_cache, mag); 1034 } 1035 } 1036 1037 OSL_TRACE( 1038 "rtl_cache_deactivate(\"%s\"): " 1039 "[slab]: allocs: %"PRIu64", frees: %"PRIu64"; total: %lu, used: %lu; " 1040 "[cpu]: allocs: %"PRIu64", frees: %"PRIu64"; " 1041 "[total]: allocs: %"PRIu64", frees: %"PRIu64"", 1042 cache->m_name, 1043 cache->m_slab_stats.m_alloc, cache->m_slab_stats.m_free, 1044 cache->m_slab_stats.m_mem_total, cache->m_slab_stats.m_mem_alloc, 1045 cache->m_cpu_stats.m_alloc, cache->m_cpu_stats.m_free, 1046 cache->m_slab_stats.m_alloc + cache->m_cpu_stats.m_alloc, 1047 cache->m_slab_stats.m_free + cache->m_cpu_stats.m_free 1048 ); 1049 1050 /* cleanup slab layer */ 1051 if (cache->m_slab_stats.m_alloc > cache->m_slab_stats.m_free) 1052 { 1053 OSL_TRACE( 1054 "rtl_cache_deactivate(\"%s\"): " 1055 "cleaning up %"PRIu64" leaked buffer(s) [%lu bytes] [%lu total]", 1056 cache->m_name, 1057 cache->m_slab_stats.m_alloc - cache->m_slab_stats.m_free, 1058 cache->m_slab_stats.m_mem_alloc, cache->m_slab_stats.m_mem_total 1059 ); 1060 1061 if (cache->m_features & RTL_CACHE_FEATURE_HASH) 1062 { 1063 /* cleanup bufctl(s) for leaking buffer(s) */ 1064 sal_Size i, n = cache->m_hash_size; 1065 for (i = 0; i < n; i++) 1066 { 1067 rtl_cache_bufctl_type * bufctl; 1068 while ((bufctl = cache->m_hash_table[i]) != NULL) 1069 { 1070 /* pop from hash table */ 1071 cache->m_hash_table[i] = bufctl->m_next, bufctl->m_next = NULL; 1072 1073 /* return to bufctl cache */ 1074 rtl_cache_free (gp_cache_bufctl_cache, bufctl); 1075 } 1076 } 1077 } 1078 { 1079 /* force cleanup of remaining slabs */ 1080 rtl_cache_slab_type *head, *slab; 1081 1082 head = &(cache->m_used_head); 1083 for (slab = head->m_slab_next; slab != head; slab = head->m_slab_next) 1084 { 1085 /* remove from 'used' queue */ 1086 QUEUE_REMOVE_NAMED(slab, slab_); 1087 1088 /* update stats */ 1089 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 1090 1091 /* free slab */ 1092 rtl_cache_slab_destroy (cache, slab); 1093 } 1094 1095 head = &(cache->m_free_head); 1096 for (slab = head->m_slab_next; slab != head; slab = head->m_slab_next) 1097 { 1098 /* remove from 'free' queue */ 1099 QUEUE_REMOVE_NAMED(slab, slab_); 1100 1101 /* update stats */ 1102 cache->m_slab_stats.m_mem_total -= cache->m_slab_size; 1103 1104 /* free slab */ 1105 rtl_cache_slab_destroy (cache, slab); 1106 } 1107 } 1108 } 1109 1110 if (cache->m_hash_table != cache->m_hash_table_0) 1111 { 1112 rtl_arena_free ( 1113 gp_cache_arena, 1114 cache->m_hash_table, 1115 cache->m_hash_size * sizeof(rtl_cache_bufctl_type*)); 1116 1117 cache->m_hash_table = cache->m_hash_table_0; 1118 cache->m_hash_size = RTL_CACHE_HASH_SIZE; 1119 cache->m_hash_shift = highbit(cache->m_hash_size) - 1; 1120 } 1121 } 1122 1123 /* ================================================================= * 1124 * 1125 * cache implementation. 1126 * 1127 * ================================================================= */ 1128 1129 /** rtl_cache_create() 1130 */ 1131 rtl_cache_type * 1132 SAL_CALL rtl_cache_create ( 1133 const char * name, 1134 sal_Size objsize, 1135 sal_Size objalign, 1136 int (SAL_CALL * constructor)(void * obj, void * userarg), 1137 void (SAL_CALL * destructor) (void * obj, void * userarg), 1138 void (SAL_CALL * reclaim) (void * userarg), 1139 void * userarg, 1140 rtl_arena_type * source, 1141 int flags 1142 ) SAL_THROW_EXTERN_C() 1143 { 1144 rtl_cache_type * result = 0; 1145 sal_Size size = sizeof(rtl_cache_type); 1146 1147 try_alloc: 1148 result = (rtl_cache_type*)rtl_arena_alloc (gp_cache_arena, &size); 1149 if (result != 0) 1150 { 1151 rtl_cache_type * cache = result; 1152 VALGRIND_CREATE_MEMPOOL(cache, 0, 0); 1153 (void) rtl_cache_constructor (cache); 1154 1155 if (!source) 1156 { 1157 /* use default arena */ 1158 OSL_ASSERT(gp_default_arena != 0); 1159 source = gp_default_arena; 1160 } 1161 1162 result = rtl_cache_activate ( 1163 cache, 1164 name, 1165 objsize, 1166 objalign, 1167 constructor, 1168 destructor, 1169 reclaim, 1170 userarg, 1171 source, 1172 flags 1173 ); 1174 1175 if (result == 0) 1176 { 1177 /* activation failed */ 1178 rtl_cache_deactivate (cache); 1179 rtl_cache_destructor (cache); 1180 VALGRIND_DESTROY_MEMPOOL(cache); 1181 rtl_arena_free (gp_cache_arena, cache, size); 1182 } 1183 } 1184 else if (gp_cache_arena == 0) 1185 { 1186 if (rtl_cache_init()) 1187 { 1188 /* try again */ 1189 goto try_alloc; 1190 } 1191 } 1192 return (result); 1193 } 1194 1195 /** rtl_cache_destroy() 1196 */ 1197 void SAL_CALL rtl_cache_destroy ( 1198 rtl_cache_type * cache 1199 ) SAL_THROW_EXTERN_C() 1200 { 1201 if (cache != 0) 1202 { 1203 rtl_cache_deactivate (cache); 1204 rtl_cache_destructor (cache); 1205 VALGRIND_DESTROY_MEMPOOL(cache); 1206 rtl_arena_free (gp_cache_arena, cache, sizeof(rtl_cache_type)); 1207 } 1208 } 1209 1210 /** rtl_cache_alloc() 1211 */ 1212 void * 1213 SAL_CALL rtl_cache_alloc ( 1214 rtl_cache_type * cache 1215 ) SAL_THROW_EXTERN_C() 1216 { 1217 void * obj = 0; 1218 1219 if (cache == 0) 1220 return (0); 1221 1222 if (cache->m_cpu_curr != 0) 1223 { 1224 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1225 1226 for (;;) 1227 { 1228 /* take object from magazine layer */ 1229 rtl_cache_magazine_type *curr, *prev, *temp; 1230 1231 curr = cache->m_cpu_curr; 1232 if ((curr != 0) && (curr->m_mag_used > 0)) 1233 { 1234 obj = curr->m_objects[--curr->m_mag_used]; 1235 #if defined(HAVE_VALGRIND_MEMCHECK_H) 1236 VALGRIND_MEMPOOL_ALLOC(cache, obj, cache->m_type_size); 1237 if (cache->m_constructor != 0) 1238 { 1239 /* keep constructed object defined */ 1240 VALGRIND_MAKE_MEM_DEFINED(obj, cache->m_type_size); 1241 } 1242 #endif /* HAVE_VALGRIND_MEMCHECK_H */ 1243 cache->m_cpu_stats.m_alloc += 1; 1244 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1245 1246 return (obj); 1247 } 1248 1249 prev = cache->m_cpu_prev; 1250 if ((prev != 0) && (prev->m_mag_used > 0)) 1251 { 1252 temp = cache->m_cpu_curr; 1253 cache->m_cpu_curr = cache->m_cpu_prev; 1254 cache->m_cpu_prev = temp; 1255 1256 continue; 1257 } 1258 1259 temp = rtl_cache_depot_exchange_alloc (cache, prev); 1260 if (temp != 0) 1261 { 1262 cache->m_cpu_prev = cache->m_cpu_curr; 1263 cache->m_cpu_curr = temp; 1264 1265 continue; 1266 } 1267 1268 /* no full magazine: fall through to slab layer */ 1269 break; 1270 } 1271 1272 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1273 } 1274 1275 /* alloc buffer from slab layer */ 1276 obj = rtl_cache_slab_alloc (cache); 1277 if ((obj != 0) && (cache->m_constructor != 0)) 1278 { 1279 /* construct object */ 1280 if (!((cache->m_constructor)(obj, cache->m_userarg))) 1281 { 1282 /* construction failure */ 1283 rtl_cache_slab_free (cache, obj), obj = 0; 1284 } 1285 } 1286 return (obj); 1287 } 1288 1289 /** rtl_cache_free() 1290 */ 1291 void 1292 SAL_CALL rtl_cache_free ( 1293 rtl_cache_type * cache, 1294 void * obj 1295 ) SAL_THROW_EXTERN_C() 1296 { 1297 if ((obj != 0) && (cache != 0)) 1298 { 1299 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1300 1301 for (;;) 1302 { 1303 /* return object to magazine layer */ 1304 rtl_cache_magazine_type *curr, *prev, *temp; 1305 1306 curr = cache->m_cpu_curr; 1307 if ((curr != 0) && (curr->m_mag_used < curr->m_mag_size)) 1308 { 1309 curr->m_objects[curr->m_mag_used++] = obj; 1310 #if defined(HAVE_VALGRIND_MEMCHECK_H) 1311 VALGRIND_MEMPOOL_FREE(cache, obj); 1312 #endif /* HAVE_VALGRIND_MEMCHECK_H */ 1313 cache->m_cpu_stats.m_free += 1; 1314 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1315 1316 return; 1317 } 1318 1319 prev = cache->m_cpu_prev; 1320 if ((prev != 0) && (prev->m_mag_used == 0)) 1321 { 1322 temp = cache->m_cpu_curr; 1323 cache->m_cpu_curr = cache->m_cpu_prev; 1324 cache->m_cpu_prev = temp; 1325 1326 continue; 1327 } 1328 1329 temp = rtl_cache_depot_exchange_free (cache, prev); 1330 if (temp != 0) 1331 { 1332 cache->m_cpu_prev = cache->m_cpu_curr; 1333 cache->m_cpu_curr = temp; 1334 1335 continue; 1336 } 1337 1338 if (rtl_cache_depot_populate(cache) != 0) 1339 { 1340 continue; 1341 } 1342 1343 /* no empty magazine: fall through to slab layer */ 1344 break; 1345 } 1346 1347 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1348 1349 /* no space for constructed object in magazine layer */ 1350 if (cache->m_destructor != 0) 1351 { 1352 /* destruct object */ 1353 (cache->m_destructor)(obj, cache->m_userarg); 1354 } 1355 1356 /* return buffer to slab layer */ 1357 rtl_cache_slab_free (cache, obj); 1358 } 1359 } 1360 1361 /* ================================================================= * 1362 * 1363 * cache wsupdate (machdep) internals. 1364 * 1365 * ================================================================= */ 1366 1367 /** rtl_cache_wsupdate_init() 1368 * 1369 * @precond g_cache_list.m_lock initialized 1370 */ 1371 static void 1372 rtl_cache_wsupdate_init (void); 1373 1374 1375 /** rtl_cache_wsupdate_wait() 1376 * 1377 * @precond g_cache_list.m_lock acquired 1378 */ 1379 static void 1380 rtl_cache_wsupdate_wait ( 1381 unsigned int seconds 1382 ); 1383 1384 /** rtl_cache_wsupdate_fini() 1385 * 1386 */ 1387 static void 1388 rtl_cache_wsupdate_fini (void); 1389 1390 /* ================================================================= */ 1391 1392 #if defined(SAL_UNX) 1393 1394 #include <sys/time.h> 1395 1396 static void * 1397 rtl_cache_wsupdate_all (void * arg); 1398 1399 static void 1400 rtl_cache_wsupdate_init (void) 1401 { 1402 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1403 g_cache_list.m_update_done = 0; 1404 (void) pthread_cond_init (&(g_cache_list.m_update_cond), NULL); 1405 if (pthread_create ( 1406 &(g_cache_list.m_update_thread), NULL, rtl_cache_wsupdate_all, (void*)(10)) != 0) 1407 { 1408 /* failure */ 1409 g_cache_list.m_update_thread = (pthread_t)(0); 1410 } 1411 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1412 } 1413 1414 static void 1415 rtl_cache_wsupdate_wait (unsigned int seconds) 1416 { 1417 if (seconds > 0) 1418 { 1419 struct timeval now; 1420 struct timespec wakeup; 1421 1422 gettimeofday(&now, 0); 1423 wakeup.tv_sec = now.tv_sec + (seconds); 1424 wakeup.tv_nsec = now.tv_usec * 1000; 1425 1426 (void) pthread_cond_timedwait ( 1427 &(g_cache_list.m_update_cond), 1428 &(g_cache_list.m_lock), 1429 &wakeup); 1430 } 1431 } 1432 1433 static void 1434 rtl_cache_wsupdate_fini (void) 1435 { 1436 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1437 g_cache_list.m_update_done = 1; 1438 pthread_cond_signal (&(g_cache_list.m_update_cond)); 1439 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1440 1441 if (g_cache_list.m_update_thread != (pthread_t)(0)) 1442 pthread_join (g_cache_list.m_update_thread, NULL); 1443 } 1444 1445 /* ================================================================= */ 1446 1447 #elif defined(SAL_OS2) 1448 1449 static void 1450 rtl_cache_wsupdate_all (void * arg); 1451 1452 static void rtl_cache_fini (void); 1453 1454 static void 1455 rtl_cache_wsupdate_init (void) 1456 { 1457 ULONG ulThreadId; 1458 APIRET rc; 1459 1460 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1461 g_cache_list.m_update_done = 0; 1462 1463 // we use atexit() because this allows CRT exit to process handler before 1464 // threads are killed. Otherwise with __attribute__(destructor) this 1465 // function is called when DosExit starts processing DLL destruction 1466 // which happens after ALL threads have been killed... 1467 atexit( rtl_cache_fini); 1468 1469 //g_cache_list.m_update_cond = CreateEvent (0, TRUE, FALSE, 0); 1470 /* Warp3 FP29 or Warp4 FP4 or better required */ 1471 rc = DosCreateEventSem( NULL, &g_cache_list.m_update_cond, 0x0800, 0); 1472 1473 g_cache_list.m_update_thread = (ULONG) _beginthread( rtl_cache_wsupdate_all, NULL, 1474 65*1024, (void*) 10); 1475 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1476 } 1477 1478 static void 1479 rtl_cache_wsupdate_wait (unsigned int seconds) 1480 { 1481 APIRET rc; 1482 if (seconds > 0) 1483 { 1484 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1485 rc = DosWaitEventSem(g_cache_list.m_update_cond, seconds*1000); 1486 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1487 } 1488 } 1489 1490 static void 1491 rtl_cache_wsupdate_fini (void) 1492 { 1493 APIRET rc; 1494 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1495 g_cache_list.m_update_done = 1; 1496 rc = DosPostEventSem(g_cache_list.m_update_cond); 1497 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1498 rc = DosWaitThread(&g_cache_list.m_update_thread, DCWW_WAIT); 1499 } 1500 1501 /* ================================================================= */ 1502 1503 #elif defined(SAL_W32) 1504 1505 static DWORD WINAPI 1506 rtl_cache_wsupdate_all (void * arg); 1507 1508 static void 1509 rtl_cache_wsupdate_init (void) 1510 { 1511 DWORD dwThreadId; 1512 1513 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1514 g_cache_list.m_update_done = 0; 1515 g_cache_list.m_update_cond = CreateEvent (0, TRUE, FALSE, 0); 1516 1517 g_cache_list.m_update_thread = 1518 CreateThread (NULL, 0, rtl_cache_wsupdate_all, (LPVOID)(10), 0, &dwThreadId); 1519 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1520 } 1521 1522 static void 1523 rtl_cache_wsupdate_wait (unsigned int seconds) 1524 { 1525 if (seconds > 0) 1526 { 1527 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1528 WaitForSingleObject (g_cache_list.m_update_cond, (DWORD)(seconds * 1000)); 1529 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1530 } 1531 } 1532 1533 static void 1534 rtl_cache_wsupdate_fini (void) 1535 { 1536 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1537 g_cache_list.m_update_done = 1; 1538 SetEvent (g_cache_list.m_update_cond); 1539 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1540 1541 WaitForSingleObject (g_cache_list.m_update_thread, INFINITE); 1542 } 1543 1544 #endif /* SAL_UNX || SAL_W32 */ 1545 1546 /* ================================================================= */ 1547 1548 /** rtl_cache_depot_wsupdate() 1549 * update depot stats and purge excess magazines. 1550 * 1551 * @precond cache->m_depot_lock acquired 1552 */ 1553 static void 1554 rtl_cache_depot_wsupdate ( 1555 rtl_cache_type * cache, 1556 rtl_cache_depot_type * depot 1557 ) 1558 { 1559 sal_Size npurge; 1560 1561 depot->m_prev_min = depot->m_curr_min; 1562 depot->m_curr_min = depot->m_mag_count; 1563 1564 npurge = SAL_MIN(depot->m_curr_min, depot->m_prev_min); 1565 for (; npurge > 0; npurge--) 1566 { 1567 rtl_cache_magazine_type * mag = rtl_cache_depot_dequeue (depot); 1568 if (mag != NULL) 1569 { 1570 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1571 rtl_cache_magazine_clear (cache, mag); 1572 rtl_cache_free (cache->m_magazine_cache, mag); 1573 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1574 } 1575 } 1576 } 1577 1578 /** rtl_cache_wsupdate() 1579 * 1580 * @precond cache->m_depot_lock released 1581 */ 1582 static void 1583 rtl_cache_wsupdate ( 1584 rtl_cache_type * cache 1585 ) 1586 { 1587 if (cache->m_magazine_cache != 0) 1588 { 1589 RTL_MEMORY_LOCK_ACQUIRE(&(cache->m_depot_lock)); 1590 1591 OSL_TRACE( 1592 "rtl_cache_wsupdate(\"%s\") " 1593 "[depot: count, curr_min, prev_min] " 1594 "full: %lu, %lu, %lu; empty: %lu, %lu, %lu", 1595 cache->m_name, 1596 cache->m_depot_full.m_mag_count, 1597 cache->m_depot_full.m_curr_min, 1598 cache->m_depot_full.m_prev_min, 1599 cache->m_depot_empty.m_mag_count, 1600 cache->m_depot_empty.m_curr_min, 1601 cache->m_depot_empty.m_prev_min 1602 ); 1603 1604 rtl_cache_depot_wsupdate (cache, &(cache->m_depot_full)); 1605 rtl_cache_depot_wsupdate (cache, &(cache->m_depot_empty)); 1606 1607 RTL_MEMORY_LOCK_RELEASE(&(cache->m_depot_lock)); 1608 } 1609 } 1610 1611 /** rtl_cache_wsupdate_all() 1612 * 1613 */ 1614 #if defined(SAL_UNX) 1615 static void * 1616 #elif defined(SAL_OS2) 1617 static void 1618 #elif defined(SAL_W32) 1619 static DWORD WINAPI 1620 #endif /* SAL_UNX || SAL_W32 */ 1621 rtl_cache_wsupdate_all (void * arg) 1622 { 1623 unsigned int seconds = (unsigned int)SAL_INT_CAST(sal_uIntPtr, arg); 1624 1625 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1626 while (!g_cache_list.m_update_done) 1627 { 1628 rtl_cache_wsupdate_wait (seconds); 1629 if (!g_cache_list.m_update_done) 1630 { 1631 rtl_cache_type * head, * cache; 1632 1633 head = &(g_cache_list.m_cache_head); 1634 for (cache = head->m_cache_next; 1635 cache != head; 1636 cache = cache->m_cache_next) 1637 { 1638 rtl_cache_wsupdate (cache); 1639 } 1640 } 1641 } 1642 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1643 1644 #if !defined(SAL_OS2) 1645 return (0); 1646 #endif 1647 } 1648 1649 /* ================================================================= * 1650 * 1651 * cache initialization. 1652 * 1653 * ================================================================= */ 1654 1655 static void 1656 rtl_cache_once_init (void) 1657 { 1658 { 1659 /* list of caches */ 1660 RTL_MEMORY_LOCK_INIT(&(g_cache_list.m_lock)); 1661 (void) rtl_cache_constructor (&(g_cache_list.m_cache_head)); 1662 } 1663 { 1664 /* cache: internal arena */ 1665 OSL_ASSERT(gp_cache_arena == NULL); 1666 1667 gp_cache_arena = rtl_arena_create ( 1668 "rtl_cache_internal_arena", 1669 64, /* quantum */ 1670 0, /* no quantum caching */ 1671 NULL, /* default source */ 1672 rtl_arena_alloc, 1673 rtl_arena_free, 1674 0 /* flags */ 1675 ); 1676 OSL_ASSERT(gp_cache_arena != NULL); 1677 1678 /* check 'gp_default_arena' initialization */ 1679 OSL_ASSERT(gp_default_arena != NULL); 1680 } 1681 { 1682 /* cache: magazine cache */ 1683 static rtl_cache_type g_cache_magazine_cache; 1684 1685 OSL_ASSERT(gp_cache_magazine_cache == NULL); 1686 VALGRIND_CREATE_MEMPOOL(&g_cache_magazine_cache, 0, 0); 1687 (void) rtl_cache_constructor (&g_cache_magazine_cache); 1688 1689 gp_cache_magazine_cache = rtl_cache_activate ( 1690 &g_cache_magazine_cache, 1691 "rtl_cache_magazine_cache", 1692 sizeof(rtl_cache_magazine_type), /* objsize */ 1693 0, /* objalign */ 1694 rtl_cache_magazine_constructor, 1695 rtl_cache_magazine_destructor, 1696 0, /* reclaim */ 1697 0, /* userarg: NYI */ 1698 gp_default_arena, /* source */ 1699 RTL_CACHE_FLAG_NOMAGAZINE /* during bootstrap; activated below */ 1700 ); 1701 OSL_ASSERT(gp_cache_magazine_cache != NULL); 1702 1703 /* activate magazine layer */ 1704 g_cache_magazine_cache.m_magazine_cache = gp_cache_magazine_cache; 1705 } 1706 { 1707 /* cache: slab (struct) cache */ 1708 static rtl_cache_type g_cache_slab_cache; 1709 1710 OSL_ASSERT(gp_cache_slab_cache == NULL); 1711 VALGRIND_CREATE_MEMPOOL(&g_cache_slab_cache, 0, 0); 1712 (void) rtl_cache_constructor (&g_cache_slab_cache); 1713 1714 gp_cache_slab_cache = rtl_cache_activate ( 1715 &g_cache_slab_cache, 1716 "rtl_cache_slab_cache", 1717 sizeof(rtl_cache_slab_type), /* objsize */ 1718 0, /* objalign */ 1719 rtl_cache_slab_constructor, 1720 rtl_cache_slab_destructor, 1721 0, /* reclaim */ 1722 0, /* userarg: none */ 1723 gp_default_arena, /* source */ 1724 0 /* flags: none */ 1725 ); 1726 OSL_ASSERT(gp_cache_slab_cache != NULL); 1727 } 1728 { 1729 /* cache: bufctl cache */ 1730 static rtl_cache_type g_cache_bufctl_cache; 1731 1732 OSL_ASSERT(gp_cache_bufctl_cache == NULL); 1733 VALGRIND_CREATE_MEMPOOL(&g_cache_bufctl_cache, 0, 0); 1734 (void) rtl_cache_constructor (&g_cache_bufctl_cache); 1735 1736 gp_cache_bufctl_cache = rtl_cache_activate ( 1737 &g_cache_bufctl_cache, 1738 "rtl_cache_bufctl_cache", 1739 sizeof(rtl_cache_bufctl_type), /* objsize */ 1740 0, /* objalign */ 1741 0, /* constructor */ 1742 0, /* destructor */ 1743 0, /* reclaim */ 1744 0, /* userarg */ 1745 gp_default_arena, /* source */ 1746 0 /* flags: none */ 1747 ); 1748 OSL_ASSERT(gp_cache_bufctl_cache != NULL); 1749 } 1750 1751 rtl_cache_wsupdate_init(); 1752 } 1753 1754 static int 1755 rtl_cache_init (void) 1756 { 1757 static sal_once_type g_once = SAL_ONCE_INIT; 1758 SAL_ONCE(&g_once, rtl_cache_once_init); 1759 return (gp_cache_arena != NULL); 1760 } 1761 1762 /* ================================================================= */ 1763 1764 /* 1765 Issue http://udk.openoffice.org/issues/show_bug.cgi?id=92388 1766 1767 Mac OS X does not seem to support "__cxa__atexit", thus leading 1768 to the situation that "__attribute__((destructor))__" functions 1769 (in particular "rtl_{memory|cache|arena}_fini") become called 1770 _before_ global C++ object d'tors. 1771 1772 Delegated the call to "rtl_cache_fini()" into a dummy C++ object, 1773 see alloc_fini.cxx . 1774 */ 1775 #if defined(__GNUC__) && !defined(MACOSX) && !defined(SAL_OS2) 1776 static void rtl_cache_fini (void) __attribute__((destructor)); 1777 #elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) 1778 #pragma fini(rtl_cache_fini) 1779 static void rtl_cache_fini (void); 1780 #endif /* __GNUC__ || __SUNPRO_C */ 1781 1782 void 1783 rtl_cache_fini (void) 1784 { 1785 if (gp_cache_arena != NULL) 1786 { 1787 rtl_cache_type * cache, * head; 1788 1789 rtl_cache_wsupdate_fini(); 1790 1791 if (gp_cache_bufctl_cache != NULL) 1792 { 1793 cache = gp_cache_bufctl_cache, gp_cache_bufctl_cache = NULL; 1794 rtl_cache_deactivate (cache); 1795 rtl_cache_destructor (cache); 1796 VALGRIND_DESTROY_MEMPOOL(cache); 1797 } 1798 if (gp_cache_slab_cache != NULL) 1799 { 1800 cache = gp_cache_slab_cache, gp_cache_slab_cache = NULL; 1801 rtl_cache_deactivate (cache); 1802 rtl_cache_destructor (cache); 1803 VALGRIND_DESTROY_MEMPOOL(cache); 1804 } 1805 if (gp_cache_magazine_cache != NULL) 1806 { 1807 cache = gp_cache_magazine_cache, gp_cache_magazine_cache = NULL; 1808 rtl_cache_deactivate (cache); 1809 rtl_cache_destructor (cache); 1810 VALGRIND_DESTROY_MEMPOOL(cache); 1811 } 1812 if (gp_cache_arena != NULL) 1813 { 1814 rtl_arena_destroy (gp_cache_arena); 1815 gp_cache_arena = NULL; 1816 } 1817 1818 RTL_MEMORY_LOCK_ACQUIRE(&(g_cache_list.m_lock)); 1819 head = &(g_cache_list.m_cache_head); 1820 for (cache = head->m_cache_next; cache != head; cache = cache->m_cache_next) 1821 { 1822 OSL_TRACE( 1823 "rtl_cache_fini(\"%s\") " 1824 "[slab]: allocs: %"PRIu64", frees: %"PRIu64"; total: %lu, used: %lu; " 1825 "[cpu]: allocs: %"PRIu64", frees: %"PRIu64"; " 1826 "[total]: allocs: %"PRIu64", frees: %"PRIu64"", 1827 cache->m_name, 1828 cache->m_slab_stats.m_alloc, cache->m_slab_stats.m_free, 1829 cache->m_slab_stats.m_mem_total, cache->m_slab_stats.m_mem_alloc, 1830 cache->m_cpu_stats.m_alloc, cache->m_cpu_stats.m_free, 1831 cache->m_slab_stats.m_alloc + cache->m_cpu_stats.m_alloc, 1832 cache->m_slab_stats.m_free + cache->m_cpu_stats.m_free 1833 ); 1834 } 1835 RTL_MEMORY_LOCK_RELEASE(&(g_cache_list.m_lock)); 1836 } 1837 } 1838 1839 /* ================================================================= */ 1840