| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: jemalloc.c,v 1.37 2015/01/20 18:31:25 christos Exp $ */ | | 1 | /* $NetBSD: jemalloc.c,v 1.38 2015/07/26 17:21:55 martin Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (C) 2006,2007 Jason Evans <jasone@FreeBSD.org>. | | 4 | * Copyright (C) 2006,2007 Jason Evans <jasone@FreeBSD.org>. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * Redistribution and use in source and binary forms, with or without | | 7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | | 8 | * modification, are permitted provided that the following conditions |
9 | * are met: | | 9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright | | 10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice(s), this list of conditions and the following disclaimer as | | 11 | * notice(s), this list of conditions and the following disclaimer as |
12 | * the first lines of this file unmodified other than the possible | | 12 | * the first lines of this file unmodified other than the possible |
13 | * addition of one or more copyright notices. | | 13 | * addition of one or more copyright notices. |
14 | * 2. Redistributions in binary form must reproduce the above copyright | | 14 | * 2. Redistributions in binary form must reproduce the above copyright |
| @@ -108,27 +108,27 @@ | | | @@ -108,27 +108,27 @@ |
108 | /* | | 108 | /* |
109 | * MALLOC_PRODUCTION disables assertions and statistics gathering. It also | | 109 | * MALLOC_PRODUCTION disables assertions and statistics gathering. It also |
110 | * defaults the A and J runtime options to off. These settings are appropriate | | 110 | * defaults the A and J runtime options to off. These settings are appropriate |
111 | * for production systems. | | 111 | * for production systems. |
112 | */ | | 112 | */ |
113 | #define MALLOC_PRODUCTION | | 113 | #define MALLOC_PRODUCTION |
114 | | | 114 | |
115 | #ifndef MALLOC_PRODUCTION | | 115 | #ifndef MALLOC_PRODUCTION |
116 | # define MALLOC_DEBUG | | 116 | # define MALLOC_DEBUG |
117 | #endif | | 117 | #endif |
118 | | | 118 | |
119 | #include <sys/cdefs.h> | | 119 | #include <sys/cdefs.h> |
120 | /* __FBSDID("$FreeBSD: src/lib/libc/stdlib/malloc.c,v 1.147 2007/06/15 22:00:16 jasone Exp $"); */ | | 120 | /* __FBSDID("$FreeBSD: src/lib/libc/stdlib/malloc.c,v 1.147 2007/06/15 22:00:16 jasone Exp $"); */ |
121 | __RCSID("$NetBSD: jemalloc.c,v 1.37 2015/01/20 18:31:25 christos Exp $"); | | 121 | __RCSID("$NetBSD: jemalloc.c,v 1.38 2015/07/26 17:21:55 martin Exp $"); |
122 | | | 122 | |
123 | #ifdef __FreeBSD__ | | 123 | #ifdef __FreeBSD__ |
124 | #include "libc_private.h" | | 124 | #include "libc_private.h" |
125 | #ifdef MALLOC_DEBUG | | 125 | #ifdef MALLOC_DEBUG |
126 | # define _LOCK_DEBUG | | 126 | # define _LOCK_DEBUG |
127 | #endif | | 127 | #endif |
128 | #include "spinlock.h" | | 128 | #include "spinlock.h" |
129 | #endif | | 129 | #endif |
130 | #include "namespace.h" | | 130 | #include "namespace.h" |
131 | #include <sys/mman.h> | | 131 | #include <sys/mman.h> |
132 | #include <sys/param.h> | | 132 | #include <sys/param.h> |
133 | #ifdef __FreeBSD__ | | 133 | #ifdef __FreeBSD__ |
134 | #include <sys/stddef.h> | | 134 | #include <sys/stddef.h> |
| @@ -773,40 +773,78 @@ static size_t base_mapped; | | | @@ -773,40 +773,78 @@ static size_t base_mapped; |
773 | */ | | 773 | */ |
774 | | | 774 | |
775 | /* | | 775 | /* |
776 | * Arenas that are used to service external requests. Not all elements of the | | 776 | * Arenas that are used to service external requests. Not all elements of the |
777 | * arenas array are necessarily used; arenas are created lazily as needed. | | 777 | * arenas array are necessarily used; arenas are created lazily as needed. |
778 | */ | | 778 | */ |
779 | static arena_t **arenas; | | 779 | static arena_t **arenas; |
780 | static unsigned narenas; | | 780 | static unsigned narenas; |
781 | static unsigned next_arena; | | 781 | static unsigned next_arena; |
782 | #ifdef _REENTRANT | | 782 | #ifdef _REENTRANT |
783 | static malloc_mutex_t arenas_mtx; /* Protects arenas initialization. */ | | 783 | static malloc_mutex_t arenas_mtx; /* Protects arenas initialization. */ |
784 | #endif | | 784 | #endif |
785 | | | 785 | |
786 | #ifndef NO_TLS | | | |
787 | /* | | 786 | /* |
788 | * Map of pthread_self() --> arenas[???], used for selecting an arena to use | | 787 | * Map of pthread_self() --> arenas[???], used for selecting an arena to use |
789 | * for allocations. | | 788 | * for allocations. |
790 | */ | | 789 | */ |
791 | static __thread arena_t *arenas_map; | | 790 | #ifndef NO_TLS |
792 | #define get_arenas_map() (arenas_map) | | 791 | static __thread arena_t **arenas_map; |
793 | #define set_arenas_map(x) (arenas_map = x) | | | |
794 | #else | | 792 | #else |
795 | #ifdef _REENTRANT | | 793 | static arena_t **arenas_map; |
796 | static thread_key_t arenas_map_key; | | | |
797 | #endif | | 794 | #endif |
798 | #define get_arenas_map() thr_getspecific(arenas_map_key) | | 795 | |
799 | #define set_arenas_map(x) thr_setspecific(arenas_map_key, x) | | 796 | #if !defined(NO_TLS) || !defined(_REENTRANT) |
| | | 797 | # define get_arenas_map() (arenas_map) |
| | | 798 | # define set_arenas_map(x) (arenas_map = x) |
| | | 799 | #else |
| | | 800 | |
| | | 801 | static thread_key_t arenas_map_key = -1; |
| | | 802 | |
| | | 803 | static inline arena_t ** |
| | | 804 | get_arenas_map(void) |
| | | 805 | { |
| | | 806 | if (!__isthreaded) |
| | | 807 | return arenas_map; |
| | | 808 | |
| | | 809 | if (arenas_map_key == -1) { |
| | | 810 | (void)thr_keycreate(&arenas_map_key, NULL); |
| | | 811 | if (arenas_map != NULL) { |
| | | 812 | thr_setspecific(arenas_map_key, arenas_map); |
| | | 813 | arenas_map = NULL; |
| | | 814 | } |
| | | 815 | } |
| | | 816 | |
| | | 817 | return thr_getspecific(arenas_map_key); |
| | | 818 | } |
| | | 819 | |
| | | 820 | static __inline void |
| | | 821 | set_arenas_map(arena_t **a) |
| | | 822 | { |
| | | 823 | if (!__isthreaded) { |
| | | 824 | arenas_map = a; |
| | | 825 | return; |
| | | 826 | } |
| | | 827 | |
| | | 828 | if (arenas_map_key == -1) { |
| | | 829 | (void)thr_keycreate(&arenas_map_key, NULL); |
| | | 830 | if (arenas_map != NULL) { |
| | | 831 | _DIAGASSERT(arenas_map == a); |
| | | 832 | arenas_map = NULL; |
| | | 833 | } |
| | | 834 | } |
| | | 835 | |
| | | 836 | thr_setspecific(arenas_map_key, a); |
| | | 837 | } |
800 | #endif | | 838 | #endif |
801 | | | 839 | |
802 | #ifdef MALLOC_STATS | | 840 | #ifdef MALLOC_STATS |
803 | /* Chunk statistics. */ | | 841 | /* Chunk statistics. */ |
804 | static chunk_stats_t stats_chunks; | | 842 | static chunk_stats_t stats_chunks; |
805 | #endif | | 843 | #endif |
806 | | | 844 | |
807 | /*******************************/ | | 845 | /*******************************/ |
808 | /* | | 846 | /* |
809 | * Runtime configuration options. | | 847 | * Runtime configuration options. |
810 | */ | | 848 | */ |
811 | const char *_malloc_options; | | 849 | const char *_malloc_options; |
812 | | | 850 | |
| @@ -3644,31 +3682,26 @@ malloc_init_hard(void) | | | @@ -3644,31 +3682,26 @@ malloc_init_hard(void) |
3644 | base_pages_alloc(0); | | 3682 | base_pages_alloc(0); |
3645 | #endif | | 3683 | #endif |
3646 | base_chunk_nodes = NULL; | | 3684 | base_chunk_nodes = NULL; |
3647 | malloc_mutex_init(&base_mtx); | | 3685 | malloc_mutex_init(&base_mtx); |
3648 | | | 3686 | |
3649 | if (ncpus > 1) { | | 3687 | if (ncpus > 1) { |
3650 | /* | | 3688 | /* |
3651 | * For SMP systems, create four times as many arenas as there | | 3689 | * For SMP systems, create four times as many arenas as there |
3652 | * are CPUs by default. | | 3690 | * are CPUs by default. |
3653 | */ | | 3691 | */ |
3654 | opt_narenas_lshift += 2; | | 3692 | opt_narenas_lshift += 2; |
3655 | } | | 3693 | } |
3656 | | | 3694 | |
3657 | #ifdef NO_TLS | | | |
3658 | /* Initialize arena key. */ | | | |
3659 | (void)thr_keycreate(&arenas_map_key, NULL); | | | |
3660 | #endif | | | |
3661 | | | | |
3662 | /* Determine how many arenas to use. */ | | 3695 | /* Determine how many arenas to use. */ |
3663 | narenas = ncpus; | | 3696 | narenas = ncpus; |
3664 | if (opt_narenas_lshift > 0) { | | 3697 | if (opt_narenas_lshift > 0) { |
3665 | if ((narenas << opt_narenas_lshift) > narenas) | | 3698 | if ((narenas << opt_narenas_lshift) > narenas) |
3666 | narenas <<= opt_narenas_lshift; | | 3699 | narenas <<= opt_narenas_lshift; |
3667 | /* | | 3700 | /* |
3668 | * Make sure not to exceed the limits of what base_malloc() | | 3701 | * Make sure not to exceed the limits of what base_malloc() |
3669 | * can handle. | | 3702 | * can handle. |
3670 | */ | | 3703 | */ |
3671 | if (narenas * sizeof(arena_t *) > chunksize) | | 3704 | if (narenas * sizeof(arena_t *) > chunksize) |
3672 | narenas = (unsigned)(chunksize / sizeof(arena_t *)); | | 3705 | narenas = (unsigned)(chunksize / sizeof(arena_t *)); |
3673 | } else if (opt_narenas_lshift < 0) { | | 3706 | } else if (opt_narenas_lshift < 0) { |
3674 | if ((narenas << opt_narenas_lshift) < narenas) | | 3707 | if ((narenas << opt_narenas_lshift) < narenas) |