| @@ -1625,26 +1625,29 @@ struct static_opts_s { | | | @@ -1625,26 +1625,29 @@ struct static_opts_s { |
1625 | * Whether to assert that allocations are not of size 0 (after any | | 1625 | * Whether to assert that allocations are not of size 0 (after any |
1626 | * bumping). | | 1626 | * bumping). |
1627 | */ | | 1627 | */ |
1628 | bool assert_nonempty_alloc; | | 1628 | bool assert_nonempty_alloc; |
1629 | | | 1629 | |
1630 | /* | | 1630 | /* |
1631 | * Whether or not to modify the 'result' argument to malloc in case of | | 1631 | * Whether or not to modify the 'result' argument to malloc in case of |
1632 | * error. | | 1632 | * error. |
1633 | */ | | 1633 | */ |
1634 | bool null_out_result_on_error; | | 1634 | bool null_out_result_on_error; |
1635 | /* Whether to set errno when we encounter an error condition. */ | | 1635 | /* Whether to set errno when we encounter an error condition. */ |
1636 | bool set_errno_on_error; | | 1636 | bool set_errno_on_error; |
1637 | | | 1637 | |
| | | 1638 | /* Whether the alignment must divide the size. */ |
| | | 1639 | bool alignment_must_divide_size; |
| | | 1640 | |
1638 | /* | | 1641 | /* |
1639 | * The minimum valid alignment for functions requesting aligned storage. | | 1642 | * The minimum valid alignment for functions requesting aligned storage. |
1640 | */ | | 1643 | */ |
1641 | size_t min_alignment; | | 1644 | size_t min_alignment; |
1642 | | | 1645 | |
1643 | /* The error string to use if we oom. */ | | 1646 | /* The error string to use if we oom. */ |
1644 | const char *oom_string; | | 1647 | const char *oom_string; |
1645 | /* The error string to use if the passed-in alignment is invalid. */ | | 1648 | /* The error string to use if the passed-in alignment is invalid. */ |
1646 | const char *invalid_alignment_string; | | 1649 | const char *invalid_alignment_string; |
1647 | | | 1650 | |
1648 | /* | | 1651 | /* |
1649 | * False if we're configured to skip some time-consuming operations. | | 1652 | * False if we're configured to skip some time-consuming operations. |
1650 | * | | 1653 | * |
| @@ -1652,26 +1655,27 @@ struct static_opts_s { | | | @@ -1652,26 +1655,27 @@ struct static_opts_s { |
1652 | * summary of several other static (or at least, static after program | | 1655 | * summary of several other static (or at least, static after program |
1653 | * initialization) options. | | 1656 | * initialization) options. |
1654 | */ | | 1657 | */ |
1655 | bool slow; | | 1658 | bool slow; |
1656 | }; | | 1659 | }; |
1657 | | | 1660 | |
1658 | JEMALLOC_ALWAYS_INLINE void | | 1661 | JEMALLOC_ALWAYS_INLINE void |
1659 | static_opts_init(static_opts_t *static_opts) { | | 1662 | static_opts_init(static_opts_t *static_opts) { |
1660 | static_opts->may_overflow = false; | | 1663 | static_opts->may_overflow = false; |
1661 | static_opts->bump_empty_alloc = false; | | 1664 | static_opts->bump_empty_alloc = false; |
1662 | static_opts->assert_nonempty_alloc = false; | | 1665 | static_opts->assert_nonempty_alloc = false; |
1663 | static_opts->null_out_result_on_error = false; | | 1666 | static_opts->null_out_result_on_error = false; |
1664 | static_opts->set_errno_on_error = false; | | 1667 | static_opts->set_errno_on_error = false; |
| | | 1668 | static_opts->alignment_must_divide_size = false; |
1665 | static_opts->min_alignment = 0; | | 1669 | static_opts->min_alignment = 0; |
1666 | static_opts->oom_string = ""; | | 1670 | static_opts->oom_string = ""; |
1667 | static_opts->invalid_alignment_string = ""; | | 1671 | static_opts->invalid_alignment_string = ""; |
1668 | static_opts->slow = false; | | 1672 | static_opts->slow = false; |
1669 | } | | 1673 | } |
1670 | | | 1674 | |
1671 | /* | | 1675 | /* |
1672 | * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we | | 1676 | * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we |
1673 | * should have one constant here per magic value there. Note however that the | | 1677 | * should have one constant here per magic value there. Note however that the |
1674 | * representations need not be related. | | 1678 | * representations need not be related. |
1675 | */ | | 1679 | */ |
1676 | #define TCACHE_IND_NONE ((unsigned)-1) | | 1680 | #define TCACHE_IND_NONE ((unsigned)-1) |
1677 | #define TCACHE_IND_AUTOMATIC ((unsigned)-2) | | 1681 | #define TCACHE_IND_AUTOMATIC ((unsigned)-2) |
| @@ -1847,26 +1851,31 @@ imalloc_body(static_opts_t *sopts, dynam | | | @@ -1847,26 +1851,31 @@ imalloc_body(static_opts_t *sopts, dynam |
1847 | if (unlikely(size == 0)) { | | 1851 | if (unlikely(size == 0)) { |
1848 | size = 1; | | 1852 | size = 1; |
1849 | } | | 1853 | } |
1850 | } | | 1854 | } |
1851 | | | 1855 | |
1852 | if (sopts->assert_nonempty_alloc) { | | 1856 | if (sopts->assert_nonempty_alloc) { |
1853 | assert (size != 0); | | 1857 | assert (size != 0); |
1854 | } | | 1858 | } |
1855 | | | 1859 | |
1856 | if (unlikely(dopts->alignment < sopts->min_alignment | | 1860 | if (unlikely(dopts->alignment < sopts->min_alignment |
1857 | || (dopts->alignment & (dopts->alignment - 1)) != 0)) { | | 1861 | || (dopts->alignment & (dopts->alignment - 1)) != 0)) { |
1858 | goto label_invalid_alignment; | | 1862 | goto label_invalid_alignment; |
1859 | } | | 1863 | } |
| | | 1864 | if (sopts->alignment_must_divide_size) { |
| | | 1865 | if (unlikely(dopts->item_size % dopts->alignment)) { |
| | | 1866 | goto label_invalid_alignment; |
| | | 1867 | } |
| | | 1868 | } |
1860 | | | 1869 | |
1861 | /* This is the beginning of the "core" algorithm. */ | | 1870 | /* This is the beginning of the "core" algorithm. */ |
1862 | | | 1871 | |
1863 | if (dopts->alignment == 0) { | | 1872 | if (dopts->alignment == 0) { |
1864 | ind = sz_size2index(size); | | 1873 | ind = sz_size2index(size); |
1865 | if (unlikely(ind >= NSIZES)) { | | 1874 | if (unlikely(ind >= NSIZES)) { |
1866 | goto label_oom; | | 1875 | goto label_oom; |
1867 | } | | 1876 | } |
1868 | if (config_stats || (config_prof && opt_prof)) { | | 1877 | if (config_stats || (config_prof && opt_prof)) { |
1869 | usize = sz_index2size(ind); | | 1878 | usize = sz_index2size(ind); |
1870 | assert(usize > 0 && usize <= LARGE_MAXCLASS); | | 1879 | assert(usize > 0 && usize <= LARGE_MAXCLASS); |
1871 | } | | 1880 | } |
1872 | } else { | | 1881 | } else { |
| @@ -2115,26 +2124,27 @@ je_aligned_alloc(size_t alignment, size_ | | | @@ -2115,26 +2124,27 @@ je_aligned_alloc(size_t alignment, size_ |
2115 | | | 2124 | |
2116 | static_opts_t sopts; | | 2125 | static_opts_t sopts; |
2117 | dynamic_opts_t dopts; | | 2126 | dynamic_opts_t dopts; |
2118 | | | 2127 | |
2119 | LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", | | 2128 | LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", |
2120 | alignment, size); | | 2129 | alignment, size); |
2121 | | | 2130 | |
2122 | static_opts_init(&sopts); | | 2131 | static_opts_init(&sopts); |
2123 | dynamic_opts_init(&dopts); | | 2132 | dynamic_opts_init(&dopts); |
2124 | | | 2133 | |
2125 | sopts.bump_empty_alloc = true; | | 2134 | sopts.bump_empty_alloc = true; |
2126 | sopts.null_out_result_on_error = true; | | 2135 | sopts.null_out_result_on_error = true; |
2127 | sopts.set_errno_on_error = true; | | 2136 | sopts.set_errno_on_error = true; |
| | | 2137 | sopts.alignment_must_divide_size = true; |
2128 | sopts.min_alignment = 1; | | 2138 | sopts.min_alignment = 1; |
2129 | sopts.oom_string = | | 2139 | sopts.oom_string = |
2130 | "<jemalloc>: Error allocating aligned memory: out of memory\n"; | | 2140 | "<jemalloc>: Error allocating aligned memory: out of memory\n"; |
2131 | sopts.invalid_alignment_string = | | 2141 | sopts.invalid_alignment_string = |
2132 | "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; | | 2142 | "<jemalloc>: Error allocating aligned memory: invalid alignment\n"; |
2133 | | | 2143 | |
2134 | dopts.result = &ret; | | 2144 | dopts.result = &ret; |
2135 | dopts.num_items = 1; | | 2145 | dopts.num_items = 1; |
2136 | dopts.item_size = size; | | 2146 | dopts.item_size = size; |
2137 | dopts.alignment = alignment; | | 2147 | dopts.alignment = alignment; |
2138 | | | 2148 | |
2139 | imalloc(&sopts, &dopts); | | 2149 | imalloc(&sopts, &dopts); |
2140 | | | 2150 | |