7
7
8
8
#include " hermes/VM/StorageProvider.h"
9
9
10
+ #include " hermes/ADT/BitArray.h"
10
11
#include " hermes/Support/CheckedMalloc.h"
11
12
#include " hermes/Support/Compiler.h"
12
13
#include " hermes/Support/OSCompat.h"
13
14
#include " hermes/VM/AlignedHeapSegment.h"
14
15
16
+ #include " llvh/ADT/BitVector.h"
15
17
#include " llvh/ADT/DenseMap.h"
16
18
#include " llvh/Support/ErrorHandling.h"
17
19
#include " llvh/Support/MathExtras.h"
@@ -55,14 +57,17 @@ namespace vm {
55
57
56
58
namespace {
57
59
60
+ // / Minimum segment storage size. Any larger segment size should be a multiple
61
+ // / of it.
62
+ constexpr auto kSegmentUnitSize = AlignedHeapSegment::kSegmentUnitSize ;
63
+
58
64
bool isAligned (void *p) {
59
- return (reinterpret_cast <uintptr_t >(p) &
60
- (FixedSizeHeapSegment::storageSize () - 1 )) == 0 ;
65
+ return (reinterpret_cast <uintptr_t >(p) & (kSegmentUnitSize - 1 )) == 0 ;
61
66
}
62
67
63
68
char *alignAlloc (void *p) {
64
- return reinterpret_cast <char *>(llvh::alignTo (
65
- reinterpret_cast <uintptr_t >(p), FixedSizeHeapSegment::storageSize () ));
69
+ return reinterpret_cast <char *>(
70
+ llvh::alignTo ( reinterpret_cast <uintptr_t >(p), kSegmentUnitSize ));
66
71
}
67
72
68
73
void *getMmapHint () {
@@ -78,68 +83,103 @@ void *getMmapHint() {
78
83
79
84
class VMAllocateStorageProvider final : public StorageProvider {
80
85
public:
81
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override ;
82
- void deleteStorageImpl (void *storage) override ;
86
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override ;
87
+ void deleteStorageImpl (void *storage, size_t sz ) override ;
83
88
};
84
89
85
90
class ContiguousVAStorageProvider final : public StorageProvider {
86
91
public:
87
92
ContiguousVAStorageProvider (size_t size)
88
- : size_(llvh::alignTo<FixedSizeHeapSegment::storageSize()>(size)) {
89
- auto result = oscompat::vm_reserve_aligned (
90
- size_, FixedSizeHeapSegment::storageSize (), getMmapHint ());
93
+ : size_(llvh::alignTo<kSegmentUnitSize >(size)),
94
+ statusBits_ (size_ / kSegmentUnitSize ) {
95
+ auto result =
96
+ oscompat::vm_reserve_aligned (size_, kSegmentUnitSize , getMmapHint ());
91
97
if (!result)
92
98
hermes_fatal (" Contiguous storage allocation failed." , result.getError ());
93
- level_ = start_ = static_cast <char *>(*result);
99
+ start_ = static_cast <char *>(*result);
94
100
oscompat::vm_name (start_, size_, kFreeRegionName );
95
101
}
96
102
~ContiguousVAStorageProvider () override {
97
103
oscompat::vm_release_aligned (start_, size_);
98
104
}
99
105
100
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override {
106
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override {
107
+ // No available space to use.
108
+ if (LLVM_UNLIKELY (firstFreeBit_ == -1 )) {
109
+ return make_error_code (OOMError::MaxStorageReached);
110
+ }
111
+
112
+ assert (
113
+ statusBits_.find_first_unset () == firstFreeBit_ &&
114
+ " firstFreeBit_ should always be the first unset bit" );
115
+
101
116
void *storage;
102
- if (!freelist_.empty ()) {
103
- storage = freelist_.back ();
104
- freelist_.pop_back ();
105
- } else if (level_ < start_ + size_) {
106
- storage =
107
- std::exchange (level_, level_ + FixedSizeHeapSegment::storageSize ());
108
- } else {
117
+ int numUnits = sz / kSegmentUnitSize ;
118
+ int nextUsedBit = statusBits_.find_next (firstFreeBit_);
119
+ int curFreeBit = firstFreeBit_;
120
+ // Search for a large enough continuous bit range.
121
+ while (nextUsedBit != -1 && (nextUsedBit - curFreeBit < numUnits)) {
122
+ curFreeBit = statusBits_.find_next_unset (nextUsedBit);
123
+ if (curFreeBit == -1 ) {
124
+ return make_error_code (OOMError::MaxStorageReached);
125
+ }
126
+ nextUsedBit = statusBits_.find_next (curFreeBit);
127
+ }
128
+ // nextUsedBit could be -1, so check if there is enough space left.
129
+ if (nextUsedBit == -1 && curFreeBit + numUnits > (int )statusBits_.size ()) {
109
130
return make_error_code (OOMError::MaxStorageReached);
110
131
}
111
- auto res =
112
- oscompat::vm_commit (storage, FixedSizeHeapSegment::storageSize ());
132
+
133
+ storage = start_ + curFreeBit * kSegmentUnitSize ;
134
+ statusBits_.set (curFreeBit, curFreeBit + numUnits);
135
+ // Reset it to the new leftmost free bit.
136
+ firstFreeBit_ = statusBits_.find_next_unset (firstFreeBit_);
137
+
138
+ auto res = oscompat::vm_commit (storage, sz);
113
139
if (res) {
114
- oscompat::vm_name (storage, FixedSizeHeapSegment::storageSize () , name);
140
+ oscompat::vm_name (storage, sz , name);
115
141
}
116
142
return res;
117
143
}
118
144
119
- void deleteStorageImpl (void *storage) override {
145
+ void deleteStorageImpl (void *storage, size_t sz ) override {
120
146
assert (
121
- !llvh::alignmentAdjustment (
122
- storage, FixedSizeHeapSegment::storageSize ()) &&
147
+ !llvh::alignmentAdjustment (storage, kSegmentUnitSize ) &&
123
148
" Storage not aligned" );
124
- assert (storage >= start_ && storage < level_ && " Storage not in region" );
125
- oscompat::vm_name (
126
- storage, FixedSizeHeapSegment::storageSize (), kFreeRegionName );
127
- oscompat::vm_uncommit (storage, FixedSizeHeapSegment::storageSize ());
128
- freelist_.push_back (storage);
149
+ assert (
150
+ storage >= start_ && storage < start_ + size_ &&
151
+ " Storage not in region" );
152
+ oscompat::vm_name (storage, sz, kFreeRegionName );
153
+ oscompat::vm_uncommit (storage, sz);
154
+ size_t numUnits = sz / kSegmentUnitSize ;
155
+ // Reset all bits for this storage.
156
+ int startIndex = (static_cast <char *>(storage) - start_) / kSegmentUnitSize ;
157
+ statusBits_.reset (startIndex, startIndex + numUnits);
158
+ if (startIndex < firstFreeBit_)
159
+ firstFreeBit_ = startIndex;
129
160
}
130
161
131
162
private:
132
163
static constexpr const char *kFreeRegionName = " hermes-free-heap" ;
133
164
size_t size_;
134
165
char *start_;
135
- char *level_;
136
- llvh::SmallVector<void *, 0 > freelist_;
166
+ // / First free bit in \c statusBits_. We always make new allocation from the
167
+ // / leftmost free bit, based on heuristics:
168
+ // / 1. Usually the reserved address space is not full.
169
+ // / 2. Storage with size kSegmentUnitSize is allocated and deleted more
170
+ // / frequently than larger storage.
171
+ // / 3. Likely small storage will find space available from leftmost free bit,
172
+ // / leaving enough space at the right side for large storage.
173
+ int firstFreeBit_{0 };
174
+ // / One bit for each kSegmentUnitSize space in the entire reserved virtual
175
+ // / address space. A bit is set if the corresponding space is used.
176
+ llvh::BitVector statusBits_;
137
177
};
138
178
139
179
class MallocStorageProvider final : public StorageProvider {
140
180
public:
141
- llvh::ErrorOr<void *> newStorageImpl (const char *name) override ;
142
- void deleteStorageImpl (void *storage) override ;
181
+ llvh::ErrorOr<void *> newStorageImpl (size_t sz, const char *name) override ;
182
+ void deleteStorageImpl (void *storage, size_t sz ) override ;
143
183
144
184
private:
145
185
// / Map aligned starts to actual starts for freeing.
@@ -149,46 +189,48 @@ class MallocStorageProvider final : public StorageProvider {
149
189
};
150
190
151
191
llvh::ErrorOr<void *> VMAllocateStorageProvider::newStorageImpl (
192
+ size_t sz,
152
193
const char *name) {
153
- assert (FixedSizeHeapSegment::storageSize () % oscompat::page_size () == 0 );
194
+ assert (kSegmentUnitSize % oscompat::page_size () == 0 );
154
195
// Allocate the space, hoping it will be the correct alignment.
155
- auto result = oscompat::vm_allocate_aligned (
156
- FixedSizeHeapSegment::storageSize (),
157
- FixedSizeHeapSegment::storageSize (),
158
- getMmapHint ());
196
+ auto result =
197
+ oscompat::vm_allocate_aligned (sz, kSegmentUnitSize , getMmapHint ());
159
198
if (!result) {
160
199
return result;
161
200
}
162
201
void *mem = *result;
163
202
assert (isAligned (mem));
164
203
(void )&isAligned;
165
- #ifdef HERMESVM_ALLOW_HUGE_PAGES
166
- oscompat::vm_hugepage (mem, FixedSizeHeapSegment::storageSize ());
167
- #endif
168
-
204
+ oscompat::vm_hugepage (mem, sz);
169
205
// Name the memory region on platforms that support naming.
170
- oscompat::vm_name (mem, FixedSizeHeapSegment::storageSize () , name);
206
+ oscompat::vm_name (mem, sz , name);
171
207
return mem;
172
208
}
173
209
174
- void VMAllocateStorageProvider::deleteStorageImpl (void *storage) {
210
+ void VMAllocateStorageProvider::deleteStorageImpl (void *storage, size_t sz ) {
175
211
if (!storage) {
176
212
return ;
177
213
}
178
- oscompat::vm_free_aligned (storage, FixedSizeHeapSegment::storageSize () );
214
+ oscompat::vm_free_aligned (storage, sz );
179
215
}
180
216
181
- llvh::ErrorOr<void *> MallocStorageProvider::newStorageImpl (const char *name) {
217
+ llvh::ErrorOr<void *> MallocStorageProvider::newStorageImpl (
218
+ size_t sz,
219
+ const char *name) {
182
220
// name is unused, can't name malloc memory.
183
221
(void )name;
184
- void *mem = checkedMalloc2 (FixedSizeHeapSegment::storageSize (), 2u );
222
+ // Allocate size of sz + kSegmentUnitSize so that we could get an address
223
+ // aligned to kSegmentUnitSize.
224
+ void *mem = checkedMalloc2 (/* count*/ 1u , sz + kSegmentUnitSize );
185
225
void *lowLim = alignAlloc (mem);
186
226
assert (isAligned (lowLim) && " New storage should be aligned" );
187
227
lowLimToAllocHandle_[lowLim] = mem;
188
228
return lowLim;
189
229
}
190
230
191
- void MallocStorageProvider::deleteStorageImpl (void *storage) {
231
+ void MallocStorageProvider::deleteStorageImpl (void *storage, size_t sz) {
232
+ // free() does not need the memory size.
233
+ (void )sz;
192
234
if (!storage) {
193
235
return ;
194
236
}
@@ -218,8 +260,11 @@ std::unique_ptr<StorageProvider> StorageProvider::mallocProvider() {
218
260
return std::unique_ptr<StorageProvider>(new MallocStorageProvider);
219
261
}
220
262
221
- llvh::ErrorOr<void *> StorageProvider::newStorage (const char *name) {
222
- auto res = newStorageImpl (name);
263
+ llvh::ErrorOr<void *> StorageProvider::newStorage (size_t sz, const char *name) {
264
+ assert (
265
+ sz && (sz % kSegmentUnitSize == 0 ) &&
266
+ " Allocated storage size must be multiples of kSegmentUnitSize" );
267
+ auto res = newStorageImpl (sz, name);
223
268
224
269
if (res) {
225
270
numSucceededAllocs_++;
@@ -230,13 +275,17 @@ llvh::ErrorOr<void *> StorageProvider::newStorage(const char *name) {
230
275
return res;
231
276
}
232
277
233
- void StorageProvider::deleteStorage (void *storage) {
278
+ void StorageProvider::deleteStorage (void *storage, size_t sz ) {
234
279
if (!storage) {
235
280
return ;
236
281
}
237
282
283
+ assert (
284
+ sz && (sz % kSegmentUnitSize == 0 ) &&
285
+ " Allocated storage size must be multiples of kSegmentUnitSize" );
286
+
238
287
numDeletedAllocs_++;
239
- deleteStorageImpl (storage);
288
+ return deleteStorageImpl (storage, sz );
240
289
}
241
290
242
291
llvh::ErrorOr<std::pair<void *, size_t >>
0 commit comments