|
145
|
1 //===-- sanitizer_allocator.cpp -------------------------------------------===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 //
|
|
|
9 // This file is shared between AddressSanitizer and ThreadSanitizer
|
|
|
10 // run-time libraries.
|
|
|
11 // This allocator is used inside run-times.
|
|
|
12 //===----------------------------------------------------------------------===//
|
|
|
13
|
|
|
14 #include "sanitizer_allocator.h"
|
|
|
15
|
|
|
16 #include "sanitizer_allocator_checks.h"
|
|
|
17 #include "sanitizer_allocator_internal.h"
|
|
|
18 #include "sanitizer_atomic.h"
|
|
|
19 #include "sanitizer_common.h"
|
|
|
20
|
|
|
21 namespace __sanitizer {
|
|
|
22
|
|
|
23 // Default allocator names.
|
|
|
24 const char *PrimaryAllocatorName = "SizeClassAllocator";
|
|
|
25 const char *SecondaryAllocatorName = "LargeMmapAllocator";
|
|
|
26
|
|
|
27 // ThreadSanitizer for Go uses libc malloc/free.
|
|
|
28 #if SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
|
|
|
29 # if SANITIZER_LINUX && !SANITIZER_ANDROID
|
|
|
30 extern "C" void *__libc_malloc(uptr size);
|
|
|
31 # if !SANITIZER_GO
|
|
|
32 extern "C" void *__libc_memalign(uptr alignment, uptr size);
|
|
|
33 # endif
|
|
|
34 extern "C" void *__libc_realloc(void *ptr, uptr size);
|
|
|
35 extern "C" void __libc_free(void *ptr);
|
|
|
36 # else
|
|
|
37 # include <stdlib.h>
|
|
|
38 # define __libc_malloc malloc
|
|
|
39 # if !SANITIZER_GO
|
|
|
40 static void *__libc_memalign(uptr alignment, uptr size) {
|
|
|
41 void *p;
|
|
|
42 uptr error = posix_memalign(&p, alignment, size);
|
|
|
43 if (error) return nullptr;
|
|
|
44 return p;
|
|
|
45 }
|
|
|
46 # endif
|
|
|
47 # define __libc_realloc realloc
|
|
|
48 # define __libc_free free
|
|
|
49 # endif
|
|
|
50
|
|
|
51 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
|
|
|
52 uptr alignment) {
|
|
|
53 (void)cache;
|
|
|
54 #if !SANITIZER_GO
|
|
|
55 if (alignment == 0)
|
|
|
56 return __libc_malloc(size);
|
|
|
57 else
|
|
|
58 return __libc_memalign(alignment, size);
|
|
|
59 #else
|
|
|
60 // Windows does not provide __libc_memalign/posix_memalign. It provides
|
|
|
61 // __aligned_malloc, but the allocated blocks can't be passed to free,
|
|
|
62 // they need to be passed to __aligned_free. InternalAlloc interface does
|
|
|
63 // not account for such requirement. Alignemnt does not seem to be used
|
|
|
64 // anywhere in runtime, so just call __libc_malloc for now.
|
|
|
65 DCHECK_EQ(alignment, 0);
|
|
|
66 return __libc_malloc(size);
|
|
|
67 #endif
|
|
|
68 }
|
|
|
69
|
|
|
70 static void *RawInternalRealloc(void *ptr, uptr size,
|
|
|
71 InternalAllocatorCache *cache) {
|
|
|
72 (void)cache;
|
|
|
73 return __libc_realloc(ptr, size);
|
|
|
74 }
|
|
|
75
|
|
|
76 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
|
|
|
77 (void)cache;
|
|
|
78 __libc_free(ptr);
|
|
|
79 }
|
|
|
80
|
|
|
81 InternalAllocator *internal_allocator() {
|
|
|
82 return 0;
|
|
|
83 }
|
|
|
84
|
|
|
85 #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
|
|
|
86
|
|
|
87 static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
|
|
|
88 static atomic_uint8_t internal_allocator_initialized;
|
|
|
89 static StaticSpinMutex internal_alloc_init_mu;
|
|
|
90
|
|
|
91 static InternalAllocatorCache internal_allocator_cache;
|
|
|
92 static StaticSpinMutex internal_allocator_cache_mu;
|
|
|
93
|
|
|
94 InternalAllocator *internal_allocator() {
|
|
|
95 InternalAllocator *internal_allocator_instance =
|
|
|
96 reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder);
|
|
|
97 if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) {
|
|
|
98 SpinMutexLock l(&internal_alloc_init_mu);
|
|
|
99 if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==
|
|
|
100 0) {
|
|
|
101 internal_allocator_instance->Init(kReleaseToOSIntervalNever);
|
|
|
102 atomic_store(&internal_allocator_initialized, 1, memory_order_release);
|
|
|
103 }
|
|
|
104 }
|
|
|
105 return internal_allocator_instance;
|
|
|
106 }
|
|
|
107
|
|
|
108 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
|
|
|
109 uptr alignment) {
|
|
|
110 if (alignment == 0) alignment = 8;
|
|
|
111 if (cache == 0) {
|
|
|
112 SpinMutexLock l(&internal_allocator_cache_mu);
|
|
|
113 return internal_allocator()->Allocate(&internal_allocator_cache, size,
|
|
|
114 alignment);
|
|
|
115 }
|
|
|
116 return internal_allocator()->Allocate(cache, size, alignment);
|
|
|
117 }
|
|
|
118
|
|
|
119 static void *RawInternalRealloc(void *ptr, uptr size,
|
|
|
120 InternalAllocatorCache *cache) {
|
|
|
121 uptr alignment = 8;
|
|
|
122 if (cache == 0) {
|
|
|
123 SpinMutexLock l(&internal_allocator_cache_mu);
|
|
|
124 return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
|
|
|
125 size, alignment);
|
|
|
126 }
|
|
|
127 return internal_allocator()->Reallocate(cache, ptr, size, alignment);
|
|
|
128 }
|
|
|
129
|
|
|
130 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
|
|
|
131 if (!cache) {
|
|
|
132 SpinMutexLock l(&internal_allocator_cache_mu);
|
|
|
133 return internal_allocator()->Deallocate(&internal_allocator_cache, ptr);
|
|
|
134 }
|
|
|
135 internal_allocator()->Deallocate(cache, ptr);
|
|
|
136 }
|
|
|
137
|
|
|
138 #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
|
|
|
139
|
|
|
140 const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
|
|
|
141
|
|
|
142 static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
|
|
|
143 SetAllocatorOutOfMemory();
|
|
|
144 Report("FATAL: %s: internal allocator is out of memory trying to allocate "
|
|
|
145 "0x%zx bytes\n", SanitizerToolName, requested_size);
|
|
|
146 Die();
|
|
|
147 }
|
|
|
148
|
|
|
149 void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
|
|
|
150 if (size + sizeof(u64) < size)
|
|
|
151 return nullptr;
|
|
|
152 void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
|
|
|
153 if (UNLIKELY(!p))
|
|
|
154 ReportInternalAllocatorOutOfMemory(size + sizeof(u64));
|
|
|
155 ((u64*)p)[0] = kBlockMagic;
|
|
|
156 return (char*)p + sizeof(u64);
|
|
|
157 }
|
|
|
158
|
|
|
159 void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
|
|
|
160 if (!addr)
|
|
|
161 return InternalAlloc(size, cache);
|
|
|
162 if (size + sizeof(u64) < size)
|
|
|
163 return nullptr;
|
|
|
164 addr = (char*)addr - sizeof(u64);
|
|
|
165 size = size + sizeof(u64);
|
|
|
166 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
|
|
|
167 void *p = RawInternalRealloc(addr, size, cache);
|
|
|
168 if (UNLIKELY(!p))
|
|
|
169 ReportInternalAllocatorOutOfMemory(size);
|
|
|
170 return (char*)p + sizeof(u64);
|
|
|
171 }
|
|
|
172
|
|
|
173 void *InternalReallocArray(void *addr, uptr count, uptr size,
|
|
|
174 InternalAllocatorCache *cache) {
|
|
|
175 if (UNLIKELY(CheckForCallocOverflow(count, size))) {
|
|
|
176 Report(
|
|
|
177 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
|
|
|
178 "cannot be represented in type size_t\n",
|
|
|
179 SanitizerToolName, count, size);
|
|
|
180 Die();
|
|
|
181 }
|
|
|
182 return InternalRealloc(addr, count * size, cache);
|
|
|
183 }
|
|
|
184
|
|
|
185 void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
|
|
|
186 if (UNLIKELY(CheckForCallocOverflow(count, size))) {
|
|
|
187 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
|
|
|
188 "cannot be represented in type size_t\n", SanitizerToolName, count,
|
|
|
189 size);
|
|
|
190 Die();
|
|
|
191 }
|
|
|
192 void *p = InternalAlloc(count * size, cache);
|
|
|
193 if (LIKELY(p))
|
|
|
194 internal_memset(p, 0, count * size);
|
|
|
195 return p;
|
|
|
196 }
|
|
|
197
|
|
|
198 void InternalFree(void *addr, InternalAllocatorCache *cache) {
|
|
|
199 if (!addr)
|
|
|
200 return;
|
|
|
201 addr = (char*)addr - sizeof(u64);
|
|
|
202 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
|
|
|
203 ((u64*)addr)[0] = 0;
|
|
|
204 RawInternalFree(addr, cache);
|
|
|
205 }
|
|
|
206
|
|
|
207 // LowLevelAllocator
|
|
|
208 constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
|
|
|
209 static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
|
|
|
210 static LowLevelAllocateCallback low_level_alloc_callback;
|
|
|
211
|
|
|
212 void *LowLevelAllocator::Allocate(uptr size) {
|
|
|
213 // Align allocation size.
|
|
|
214 size = RoundUpTo(size, low_level_alloc_min_alignment);
|
|
|
215 if (allocated_end_ - allocated_current_ < (sptr)size) {
|
|
|
216 uptr size_to_allocate = Max(size, GetPageSizeCached());
|
|
|
217 allocated_current_ =
|
|
|
218 (char*)MmapOrDie(size_to_allocate, __func__);
|
|
|
219 allocated_end_ = allocated_current_ + size_to_allocate;
|
|
|
220 if (low_level_alloc_callback) {
|
|
|
221 low_level_alloc_callback((uptr)allocated_current_,
|
|
|
222 size_to_allocate);
|
|
|
223 }
|
|
|
224 }
|
|
|
225 CHECK(allocated_end_ - allocated_current_ >= (sptr)size);
|
|
|
226 void *res = allocated_current_;
|
|
|
227 allocated_current_ += size;
|
|
|
228 return res;
|
|
|
229 }
|
|
|
230
|
|
|
231 void SetLowLevelAllocateMinAlignment(uptr alignment) {
|
|
|
232 CHECK(IsPowerOfTwo(alignment));
|
|
|
233 low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment);
|
|
|
234 }
|
|
|
235
|
|
|
236 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
|
|
|
237 low_level_alloc_callback = callback;
|
|
|
238 }
|
|
|
239
|
|
|
240 // Allocator's OOM and other errors handling support.
|
|
|
241
|
|
|
242 static atomic_uint8_t allocator_out_of_memory = {0};
|
|
|
243 static atomic_uint8_t allocator_may_return_null = {0};
|
|
|
244
|
|
|
245 bool IsAllocatorOutOfMemory() {
|
|
|
246 return atomic_load_relaxed(&allocator_out_of_memory);
|
|
|
247 }
|
|
|
248
|
|
|
249 void SetAllocatorOutOfMemory() {
|
|
|
250 atomic_store_relaxed(&allocator_out_of_memory, 1);
|
|
|
251 }
|
|
|
252
|
|
|
253 bool AllocatorMayReturnNull() {
|
|
|
254 return atomic_load(&allocator_may_return_null, memory_order_relaxed);
|
|
|
255 }
|
|
|
256
|
|
|
257 void SetAllocatorMayReturnNull(bool may_return_null) {
|
|
|
258 atomic_store(&allocator_may_return_null, may_return_null,
|
|
|
259 memory_order_relaxed);
|
|
|
260 }
|
|
|
261
|
|
|
262 void PrintHintAllocatorCannotReturnNull() {
|
|
|
263 Report("HINT: if you don't care about these errors you may set "
|
|
|
264 "allocator_may_return_null=1\n");
|
|
|
265 }
|
|
|
266
|
|
|
267 } // namespace __sanitizer
|
