7
7
8
8
#include " hermes/VM/StorageProvider.h"
9
9
10
+ #include " hermes/ADT/BitArray.h"
10
11
#include " hermes/Support/CheckedMalloc.h"
11
12
#include " hermes/Support/Compiler.h"
12
13
#include " hermes/Support/OSCompat.h"
13
14
#include " hermes/VM/AlignedHeapSegment.h"
14
15
16
+ #include " llvh/ADT/BitVector.h"
15
17
#include " llvh/ADT/DenseMap.h"
16
18
#include " llvh/Support/ErrorHandling.h"
17
19
#include " llvh/Support/MathExtras.h"
@@ -55,14 +57,17 @@ namespace vm {
55
57
56
58
namespace {
57
59
60
+ // / Minimum segment storage size. Any larger segment size should be a multiple
61
+ // / of it.
62
+ constexpr auto kSegmentUnitSize = AlignedHeapSegment::kSegmentUnitSize ;
63
+
58
64
bool isAligned (void *p) {
59
- return (reinterpret_cast <uintptr_t >(p) &
60
- (FixedSizeHeapSegment::storageSize () - 1 )) == 0 ;
65
+ return (reinterpret_cast <uintptr_t >(p) & (kSegmentUnitSize - 1 )) == 0 ;
61
66
}
62
67
63
68
char *alignAlloc (void *p) {
64
- return reinterpret_cast <char *>(llvh::alignTo (
65
- reinterpret_cast <uintptr_t >(p), FixedSizeHeapSegment::storageSize () ));
69
+ return reinterpret_cast <char *>(
70
+ llvh::alignTo ( reinterpret_cast <uintptr_t >(p), kSegmentUnitSize ));
66
71
}
67
72
68
73
void *getMmapHint () {
@@ -78,68 +83,108 @@ void *getMmapHint() {
78
83
79
84
class VMAllocateStorageProvider final : public StorageProvider {
80
85
public:
81
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override ;
82
- void deleteStorageImpl (void *storage) override ;
86
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override ;
87
+ void deleteStorageImpl (void *storage, size_t sz ) override ;
83
88
};
84
89
85
90
class ContiguousVAStorageProvider final : public StorageProvider {
86
91
public:
87
92
ContiguousVAStorageProvider (size_t size)
88
- : size_(llvh::alignTo<FixedSizeHeapSegment::storageSize()>(size)) {
89
- auto result = oscompat::vm_reserve_aligned (
90
- size_, FixedSizeHeapSegment::storageSize (), getMmapHint ());
93
+ : size_(llvh::alignTo<kSegmentUnitSize >(size)),
94
+ statusBits_ (size_ / kSegmentUnitSize ) {
95
+ auto result =
96
+ oscompat::vm_reserve_aligned (size_, kSegmentUnitSize , getMmapHint ());
91
97
if (!result)
92
98
hermes_fatal (" Contiguous storage allocation failed." , result.getError ());
93
- level_ = start_ = static_cast <char *>(*result);
99
+ start_ = static_cast <char *>(*result);
94
100
oscompat::vm_name (start_, size_, kFreeRegionName );
95
101
}
96
102
~ContiguousVAStorageProvider () override {
97
103
oscompat::vm_release_aligned (start_, size_);
98
104
}
99
105
100
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override {
106
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override {
107
+ // No available space to use.
108
+ if (LLVM_UNLIKELY (firstFreeBit_ == -1 )) {
109
+ return make_error_code (OOMError::MaxStorageReached);
110
+ }
111
+
112
+ assert (
113
+ statusBits_.find_first_unset () == firstFreeBit_ &&
114
+ " firstFreeBit_ should always be the first unset bit" );
115
+
101
116
void *storage;
102
- if (!freelist_.empty ()) {
103
- storage = freelist_.back ();
104
- freelist_.pop_back ();
105
- } else if (level_ < start_ + size_) {
106
- storage =
107
- std::exchange (level_, level_ + FixedSizeHeapSegment::storageSize ());
108
- } else {
117
+ int numUnits = sz / kSegmentUnitSize ;
118
+ int nextUsedBit = statusBits_.find_next (firstFreeBit_);
119
+ int curFreeBit = firstFreeBit_;
120
+ // Search for a large enough continuous bit range.
121
+ while (nextUsedBit != -1 && (nextUsedBit - curFreeBit < numUnits)) {
122
+ curFreeBit = statusBits_.find_next_unset (nextUsedBit);
123
+ if (curFreeBit == -1 ) {
124
+ return make_error_code (OOMError::MaxStorageReached);
125
+ }
126
+ nextUsedBit = statusBits_.find_next (curFreeBit);
127
+ }
128
+ // nextUsedBit could be -1, so check if there is enough space left.
129
+ if (nextUsedBit == -1 && curFreeBit + numUnits > (int )statusBits_.size ()) {
109
130
return make_error_code (OOMError::MaxStorageReached);
110
131
}
111
- auto res =
112
- oscompat::vm_commit (storage, FixedSizeHeapSegment::storageSize ());
132
+
133
+ storage = start_ + curFreeBit * kSegmentUnitSize ;
134
+ statusBits_.set (curFreeBit, curFreeBit + numUnits);
135
+ // Reset it to the new leftmost free bit.
136
+ // If curFreeBit != firstFreeBit_, it means the current firstFreeBit_ is
137
+ // still unset, and is certainly leftmost free bit.
138
+ if (curFreeBit == firstFreeBit_) {
139
+ // Subtracted by 1 since curFreeBit + numUnits might be the end.
140
+ firstFreeBit_ = statusBits_.find_next_unset (curFreeBit + numUnits - 1 );
141
+ }
142
+
143
+ auto res = oscompat::vm_commit (storage, sz);
113
144
if (res) {
114
- oscompat::vm_name (storage, FixedSizeHeapSegment::storageSize () , name);
145
+ oscompat::vm_name (storage, sz , name);
115
146
}
116
147
return res;
117
148
}
118
149
119
- void deleteStorageImpl (void *storage) override {
150
+ void deleteStorageImpl (void *storage, size_t sz ) override {
120
151
assert (
121
- !llvh::alignmentAdjustment (
122
- storage, FixedSizeHeapSegment::storageSize ()) &&
152
+ !llvh::alignmentAdjustment (storage, kSegmentUnitSize ) &&
123
153
" Storage not aligned" );
124
- assert (storage >= start_ && storage < level_ && " Storage not in region" );
125
- oscompat::vm_name (
126
- storage, FixedSizeHeapSegment::storageSize (), kFreeRegionName );
127
- oscompat::vm_uncommit (storage, FixedSizeHeapSegment::storageSize ());
128
- freelist_.push_back (storage);
154
+ assert (
155
+ storage >= start_ && storage < start_ + size_ &&
156
+ " Storage not in region" );
157
+ oscompat::vm_name (storage, sz, kFreeRegionName );
158
+ oscompat::vm_uncommit (storage, sz);
159
+ size_t numUnits = sz / kSegmentUnitSize ;
160
+ // Reset all bits for this storage.
161
+ int startIndex = (static_cast <char *>(storage) - start_) / kSegmentUnitSize ;
162
+ statusBits_.reset (startIndex, startIndex + numUnits);
163
+ if (startIndex < firstFreeBit_)
164
+ firstFreeBit_ = startIndex;
129
165
}
130
166
131
167
private:
132
168
static constexpr const char *kFreeRegionName = " hermes-free-heap" ;
133
169
size_t size_;
134
170
char *start_;
135
- char *level_;
136
- llvh::SmallVector<void *, 0 > freelist_;
171
+ // / First free bit in \c statusBits_. We always make new allocation from the
172
+ // / leftmost free bit, based on heuristics:
173
+ // / 1. Usually the reserved address space is not full.
174
+ // / 2. Storage with size kSegmentUnitSize is allocated and deleted more
175
+ // / frequently than larger storage.
176
+ // / 3. Likely small storage will find space available from leftmost free bit,
177
+ // / leaving enough space at the right side for large storage.
178
+ int firstFreeBit_{0 };
179
+ // / One bit for each kSegmentUnitSize space in the entire reserved virtual
180
+ // / address space. A bit is set if the corresponding space is used.
181
+ llvh::BitVector statusBits_;
137
182
};
138
183
139
184
class MallocStorageProvider final : public StorageProvider {
140
185
public:
141
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override ;
142
- void deleteStorageImpl (void *storage) override ;
186
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override ;
187
+ void deleteStorageImpl (void *storage, size_t sz ) override ;
143
188
144
189
private:
145
190
// / Map aligned starts to actual starts for freeing.
@@ -149,46 +194,48 @@ class MallocStorageProvider final : public StorageProvider {
149
194
};
150
195
151
196
llvh::ErrorOr<void *> VMAllocateStorageProvider::newStorageImpl (
197
+ size_t sz,
152
198
const char *name) {
153
- assert (FixedSizeHeapSegment::storageSize () % oscompat::page_size () == 0 );
199
+ assert (kSegmentUnitSize % oscompat::page_size () == 0 );
154
200
// Allocate the space, hoping it will be the correct alignment.
155
- auto result = oscompat::vm_allocate_aligned (
156
- FixedSizeHeapSegment::storageSize (),
157
- FixedSizeHeapSegment::storageSize (),
158
- getMmapHint ());
201
+ auto result =
202
+ oscompat::vm_allocate_aligned (sz, kSegmentUnitSize , getMmapHint ());
159
203
if (!result) {
160
204
return result;
161
205
}
162
206
void *mem = *result;
163
207
assert (isAligned (mem));
164
208
(void )&isAligned;
165
- #ifdef HERMESVM_ALLOW_HUGE_PAGES
166
- oscompat::vm_hugepage (mem, FixedSizeHeapSegment::storageSize ());
167
- #endif
168
-
209
+ oscompat::vm_hugepage (mem, sz);
169
210
// Name the memory region on platforms that support naming.
170
- oscompat::vm_name (mem, FixedSizeHeapSegment::storageSize () , name);
211
+ oscompat::vm_name (mem, sz , name);
171
212
return mem;
172
213
}
173
214
174
- void VMAllocateStorageProvider::deleteStorageImpl (void *storage) {
215
+ void VMAllocateStorageProvider::deleteStorageImpl (void *storage, size_t sz ) {
175
216
if (!storage) {
176
217
return ;
177
218
}
178
- oscompat::vm_free_aligned (storage, FixedSizeHeapSegment::storageSize () );
219
+ oscompat::vm_free_aligned (storage, sz );
179
220
}
180
221
181
- llvh::ErrorOr<void *> MallocStorageProvider::newStorageImpl (const char *name) {
222
+ llvh::ErrorOr<void *> MallocStorageProvider::newStorageImpl (
223
+ size_t sz,
224
+ const char *name) {
182
225
// name is unused, can't name malloc memory.
183
226
(void )name;
184
- void *mem = checkedMalloc2 (FixedSizeHeapSegment::storageSize (), 2u );
227
+ // Allocate size of sz + kSegmentUnitSize so that we could get an address
228
+ // aligned to kSegmentUnitSize.
229
+ void *mem = checkedMalloc2 (/* count*/ 1u , sz + kSegmentUnitSize );
185
230
void *lowLim = alignAlloc (mem);
186
231
assert (isAligned (lowLim) && " New storage should be aligned" );
187
232
lowLimToAllocHandle_[lowLim] = mem;
188
233
return lowLim;
189
234
}
190
235
191
- void MallocStorageProvider::deleteStorageImpl (void *storage) {
236
+ void MallocStorageProvider::deleteStorageImpl (void *storage, size_t sz) {
237
+ // free() does not need the memory size.
238
+ (void )sz;
192
239
if (!storage) {
193
240
return ;
194
241
}
@@ -218,8 +265,11 @@ std::unique_ptr<StorageProvider> StorageProvider::mallocProvider() {
218
265
return std::unique_ptr<StorageProvider>(new MallocStorageProvider);
219
266
}
220
267
221
- llvh::ErrorOr<void *> StorageProvider::newStorage (const char *name) {
222
- auto res = newStorageImpl (name);
268
+ llvh::ErrorOr<void *> StorageProvider::newStorage (size_t sz, const char *name) {
269
+ assert (
270
+ sz && (sz % kSegmentUnitSize == 0 ) &&
271
+ " Allocated storage size must be multiples of kSegmentUnitSize" );
272
+ auto res = newStorageImpl (sz, name);
223
273
224
274
if (res) {
225
275
numSucceededAllocs_++;
@@ -230,13 +280,17 @@ llvh::ErrorOr<void *> StorageProvider::newStorage(const char *name) {
230
280
return res;
231
281
}
232
282
233
- void StorageProvider::deleteStorage (void *storage) {
283
+ void StorageProvider::deleteStorage (void *storage, size_t sz ) {
234
284
if (!storage) {
235
285
return ;
236
286
}
237
287
288
+ assert (
289
+ sz && (sz % kSegmentUnitSize == 0 ) &&
290
+ " Allocated storage size must be multiples of kSegmentUnitSize" );
291
+
238
292
numDeletedAllocs_++;
239
- deleteStorageImpl (storage);
293
+ return deleteStorageImpl (storage, sz );
240
294
}
241
295
242
296
llvh::ErrorOr<std::pair<void *, size_t >>
0 commit comments