Skip to content

Commit d6306ef

Browse files
committedNov 1, 2023
feat: 优化主线程执行策略,提升整体性能
1 parent 4c9a7f1 commit d6306ef

13 files changed

+271
-46
lines changed
 

‎.gitattributes

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# To shield the difference between Windows and Linux systems.
2+
3+
*.h text eol=native
4+
*.cpp text eol=native
5+
*.inl text eol=native

‎src/CBasic/CStatus.h

+17-1
Original file line numberDiff line numberDiff line change
@@ -46,18 +46,34 @@ class CSTATUS {
4646
}
4747

4848
CSTATUS(const CSTATUS &status) {
49+
if (status.isOK()) {
50+
return;
51+
}
52+
4953
this->error_code_ = status.error_code_;
5054
this->error_info_ = status.error_info_;
5155
this->error_locate_ = status.error_locate_;
5256
}
5357

5458
CSTATUS(const CSTATUS &&status) noexcept {
59+
if (status.isOK()) {
60+
return;
61+
}
62+
5563
this->error_code_ = status.error_code_;
5664
this->error_info_ = status.error_info_;
5765
this->error_locate_ = status.error_locate_;
5866
}
5967

60-
CSTATUS& operator=(const CSTATUS& status) = default;
68+
CSTATUS& operator=(const CSTATUS& status) {
69+
if (!status.isOK()) {
70+
// 如果status是正常的话,则所有数据保持不变
71+
this->error_code_ = status.error_code_;
72+
this->error_info_ = status.error_info_;
73+
this->error_locate_ = status.error_locate_;
74+
}
75+
return (*this);
76+
}
6177

6278
CSTATUS& operator+=(const CSTATUS& cur) {
6379
/**

‎src/UtilsCtrl/ThreadPool/Queue/UAtomicQueue.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class UAtomicQueue : public UQueueObject {
4343
*/
4444
CBool tryPop(T& value) {
4545
CBool result = false;
46-
if (mutex_.try_lock()) {
46+
if (!queue_.empty() && mutex_.try_lock()) {
4747
if (!queue_.empty()) {
4848
value = std::move(*queue_.front());
4949
queue_.pop();
@@ -64,7 +64,7 @@ class UAtomicQueue : public UQueueObject {
6464
*/
6565
CBool tryPop(std::vector<T>& values, int maxPoolBatchSize) {
6666
CBool result = false;
67-
if (mutex_.try_lock()) {
67+
if (!queue_.empty() && mutex_.try_lock()) {
6868
while (!queue_.empty() && maxPoolBatchSize-- > 0) {
6969
values.emplace_back(std::move(*queue_.front()));
7070
queue_.pop();
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/***************************
2+
@Author: Chunel
3+
@Contact: chunel@foxmail.com
4+
@File: ULockFreeRingBufferQueue.h
5+
@Time: 2023/10/7 21:35
6+
@Desc:
7+
***************************/
8+
9+
#ifndef CGRAPH_ULOCKFREERINGBUFFERQUEUE_H
10+
#define CGRAPH_ULOCKFREERINGBUFFERQUEUE_H
11+
12+
#include <atomic>
13+
#include <memory>
14+
15+
#include "UQueueObject.h"
16+
17+
CGRAPH_NAMESPACE_BEGIN
18+
19+
template<typename T, CInt CAPACITY = 32>
20+
class ULockFreeRingBufferQueue : public UQueueObject {
21+
public:
22+
explicit ULockFreeRingBufferQueue() {
23+
head_ = 0;
24+
tail_ = 0;
25+
ring_buffer_.resize(CAPACITY);
26+
}
27+
28+
~ULockFreeRingBufferQueue() override {
29+
ring_buffer_.clear();
30+
}
31+
32+
/**
33+
* 写入一个任务
34+
* @param value
35+
*/
36+
CVoid push(T&& value) {
37+
int curTail = tail_.load(std::memory_order_relaxed);
38+
int nextTail = (curTail + 1) % CAPACITY;
39+
40+
while (nextTail == head_.load(std::memory_order_acquire)) {
41+
// 队列已满,等待其他线程出队
42+
std::this_thread::yield();
43+
}
44+
45+
ring_buffer_[curTail] = std::move(value);
46+
tail_.store(nextTail, std::memory_order_release);
47+
}
48+
49+
/**
50+
* 尝试弹出一个任务
51+
* @param value
52+
* @return
53+
*/
54+
CBool tryPop(T& value) {
55+
int curHead = head_.load(std::memory_order_relaxed);
56+
if (curHead == tail_.load(std::memory_order_acquire)) {
57+
// 队列已空,直接返回false
58+
return false;
59+
}
60+
61+
value = std::move(ring_buffer_[curHead]);
62+
63+
int nextHead = (curHead + 1) % CAPACITY;
64+
head_.store(nextHead, std::memory_order_release);
65+
return true;
66+
}
67+
68+
69+
private:
70+
std::atomic<CInt> head_; // 开始元素(较早写入的)的位置
71+
std::atomic<CInt> tail_; // 尾部的位置
72+
std::vector<std::unique_ptr<T> > ring_buffer_; // 环形队列
73+
};
74+
75+
CGRAPH_NAMESPACE_END
76+
77+
#endif //CGRAPH_ULOCKFREERINGBUFFERQUEUE_H

‎src/UtilsCtrl/ThreadPool/Queue/UQueueInclude.h

+1
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@
1313
#include "UWorkStealingQueue.h"
1414
#include "UAtomicPriorityQueue.h"
1515
#include "UAtomicRingBufferQueue.h"
16+
#include "ULockFreeRingBufferQueue.h"
1617

1718
#endif //CGRAPH_UQUEUEINCLUDE_H

‎src/UtilsCtrl/ThreadPool/Queue/UWorkStealingQueue.h

+55-19
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,20 @@
1313
#include <deque>
1414

1515
#include "UQueueObject.h"
16-
#include "../Task/UTaskInclude.h"
17-
#include "../Lock/ULockInclude.h"
1816

1917
CGRAPH_NAMESPACE_BEGIN
2018

19+
template<typename T>
2120
class UWorkStealingQueue : public UQueueObject {
2221
public:
2322
/**
2423
* 向队列中写入信息
2524
* @param task
2625
*/
27-
CVoid push(UTask&& task) {
26+
CVoid push(T&& task) {
2827
while (true) {
2928
if (lock_.try_lock()) {
30-
deque_.emplace_front(std::move(task));
29+
deque_.emplace_front(std::forward<T>(task));
3130
lock_.unlock();
3231
break;
3332
} else {
@@ -42,10 +41,47 @@ class UWorkStealingQueue : public UQueueObject {
4241
* @param task
4342
* @return
4443
*/
45-
CBool tryPush(UTask&& task) {
44+
CBool tryPush(T&& task) {
4645
CBool result = false;
4746
if (lock_.try_lock()) {
48-
deque_.emplace_back(std::move(task));
47+
deque_.emplace_back(std::forward<T>(task));
48+
lock_.unlock();
49+
result = true;
50+
}
51+
return result;
52+
}
53+
54+
55+
/**
56+
* 向队列中写入信息
57+
* @param task
58+
*/
59+
CVoid push(std::vector<T>& tasks) {
60+
while (true) {
61+
if (lock_.try_lock()) {
62+
for (const auto& task : tasks) {
63+
deque_.emplace_front(std::forward<T>(task));
64+
}
65+
lock_.unlock();
66+
break;
67+
} else {
68+
std::this_thread::yield();
69+
}
70+
}
71+
}
72+
73+
74+
/**
75+
* 尝试批量写入内容
76+
* @param tasks
77+
* @return
78+
*/
79+
CBool tryPush(std::vector<T>& tasks) {
80+
CBool result = false;
81+
if (lock_.try_lock()) {
82+
for (const auto& task : tasks) {
83+
deque_.emplace_back(std::forward<T>(task));
84+
}
4985
lock_.unlock();
5086
result = true;
5187
}
@@ -58,12 +94,12 @@ class UWorkStealingQueue : public UQueueObject {
5894
* @param task
5995
* @return
6096
*/
61-
CBool tryPop(UTask& task) {
97+
CBool tryPop(T& task) {
6298
// 这里不使用raii锁,主要是考虑到多线程的情况下,可能会重复进入
6399
bool result = false;
64-
if (lock_.try_lock()) {
100+
if (!deque_.empty() && lock_.try_lock()) {
65101
if (!deque_.empty()) {
66-
task = std::move(deque_.front()); // 从前方弹出
102+
task = std::forward<T>(deque_.front()); // 从前方弹出
67103
deque_.pop_front();
68104
result = true;
69105
}
@@ -80,11 +116,11 @@ class UWorkStealingQueue : public UQueueObject {
80116
* @param maxLocalBatchSize
81117
* @return
82118
*/
83-
CBool tryPop(UTaskArrRef taskArr, int maxLocalBatchSize) {
119+
CBool tryPop(std::vector<T>& taskArr, int maxLocalBatchSize) {
84120
bool result = false;
85-
if (lock_.try_lock()) {
121+
if (!deque_.empty() && lock_.try_lock()) {
86122
while (!deque_.empty() && maxLocalBatchSize--) {
87-
taskArr.emplace_back(std::move(deque_.front()));
123+
taskArr.emplace_back(std::forward<T>(deque_.front()));
88124
deque_.pop_front();
89125
result = true;
90126
}
@@ -100,11 +136,11 @@ class UWorkStealingQueue : public UQueueObject {
100136
* @param task
101137
* @return
102138
*/
103-
CBool trySteal(UTask& task) {
139+
CBool trySteal(T& task) {
104140
bool result = false;
105-
if (lock_.try_lock()) {
141+
if (!deque_.empty() && lock_.try_lock()) {
106142
if (!deque_.empty()) {
107-
task = std::move(deque_.back()); // 从后方窃取
143+
task = std::forward<T>(deque_.back()); // 从后方窃取
108144
deque_.pop_back();
109145
result = true;
110146
}
@@ -120,11 +156,11 @@ class UWorkStealingQueue : public UQueueObject {
120156
* @param taskArr
121157
* @return
122158
*/
123-
CBool trySteal(UTaskArrRef taskArr, int maxStealBatchSize) {
159+
CBool trySteal(std::vector<T>& taskArr, int maxStealBatchSize) {
124160
bool result = false;
125-
if (lock_.try_lock()) {
161+
if (!deque_.empty() && lock_.try_lock()) {
126162
while (!deque_.empty() && maxStealBatchSize--) {
127-
taskArr.emplace_back(std::move(deque_.back()));
163+
taskArr.emplace_back(std::forward<T>(deque_.back()));
128164
deque_.pop_back();
129165
result = true;
130166
}
@@ -139,7 +175,7 @@ class UWorkStealingQueue : public UQueueObject {
139175
CGRAPH_NO_ALLOWED_COPY(UWorkStealingQueue)
140176

141177
private:
142-
std::deque<UTask> deque_; // 存放任务的双向队列
178+
std::deque<T> deque_; // 存放任务的双向队列
143179
std::mutex lock_; // 用于处理deque_的锁
144180
};
145181

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/***************************
2+
@Author: Chunel
3+
@Contact: chunel@foxmail.com
4+
@File: USemaphore.h
5+
@Time: 2023/10/9 22:01
6+
@Desc:
7+
***************************/
8+
9+
#ifndef CGRAPH_USEMAPHORE_H
10+
#define CGRAPH_USEMAPHORE_H
11+
12+
CGRAPH_NAMESPACE_BEGIN
13+
14+
#include <mutex>
15+
#include <condition_variable>
16+
17+
#include "../UThreadObject.h"
18+
19+
class USemaphore : public UThreadObject {
20+
public:
21+
/**
22+
* 触发一次信号
23+
*/
24+
CVoid signal() {
25+
CGRAPH_UNIQUE_LOCK lk(mutex_);
26+
cnt_++;
27+
if (cnt_ <= 0) {
28+
cv_.notify_one();
29+
}
30+
}
31+
32+
/**
33+
* 等待信号触发
34+
*/
35+
CVoid wait() {
36+
CGRAPH_UNIQUE_LOCK lk(mutex_);
37+
cnt_--;
38+
if (cnt_ < 0) {
39+
cv_.wait(lk);
40+
}
41+
}
42+
43+
private:
44+
CInt cnt_ = 0; // 记录当前的次数
45+
std::mutex mutex_;
46+
std::condition_variable cv_;
47+
};
48+
49+
CGRAPH_NAMESPACE_END
50+
51+
#endif //CGRAPH_USEMAPHORE_H

‎src/UtilsCtrl/ThreadPool/Thread/UThreadPrimary.h

+52-18
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
#ifndef CGRAPH_UTHREADPRIMARY_H
1010
#define CGRAPH_UTHREADPRIMARY_H
1111

12+
#include <vector>
13+
#include <mutex>
14+
1215
#include "UThreadBase.h"
1316

1417
CGRAPH_NAMESPACE_BEGIN
@@ -17,7 +20,6 @@ class UThreadPrimary : public UThreadBase {
1720
protected:
1821
explicit UThreadPrimary() {
1922
index_ = CGRAPH_SECONDARY_THREAD_COMMON_ID;
20-
steal_range_ = 0;
2123
pool_threads_ = nullptr;
2224
type_ = CGRAPH_THREAD_TYPE_PRIMARY;
2325
}
@@ -29,7 +31,7 @@ class UThreadPrimary : public UThreadBase {
2931
CGRAPH_ASSERT_NOT_NULL(config_)
3032

3133
is_init_ = true;
32-
steal_range_ = config_->calcStealRange();
34+
buildStealTargets();
3335
thread_ = std::move(std::thread(&UThreadPrimary::run, this));
3436
setSchedParam();
3537
setAffinity(index_);
@@ -64,7 +66,7 @@ class UThreadPrimary : public UThreadBase {
6466
* 线程执行函数
6567
* @return
6668
*/
67-
CStatus run() override {
69+
CStatus run() final {
6870
CGRAPH_FUNCTION_BEGIN
6971
CGRAPH_ASSERT_INIT(true)
7072
CGRAPH_ASSERT_NOT_NULL(pool_threads_)
@@ -91,7 +93,7 @@ class UThreadPrimary : public UThreadBase {
9193
if (popTask(task) || popPoolTask(task) || stealTask(task)) {
9294
runTask(task);
9395
} else {
94-
std::this_thread::yield();
96+
fatWait();
9597
}
9698
}
9799

@@ -102,7 +104,22 @@ class UThreadPrimary : public UThreadBase {
102104
// 尝试从主线程中获取/盗取批量task,如果成功,则依次执行
103105
runTasks(tasks);
104106
} else {
105-
std::this_thread::yield();
107+
fatWait();
108+
}
109+
}
110+
111+
112+
/**
113+
* 如果总是进入无task的状态,则开始休眠
114+
* 休眠一定时间后,然后恢复执行状态,避免出现
115+
*/
116+
CVoid fatWait() {
117+
cur_empty_epoch_++;
118+
std::this_thread::yield();
119+
if (cur_empty_epoch_ >= config_->primary_thread_busy_epoch_) {
120+
CGRAPH_UNIQUE_LOCK lk(mutex_);
121+
cv_.wait_for(lk, std::chrono::milliseconds(config_->primary_thread_empty_interval_));
122+
cur_empty_epoch_ = 0;
106123
}
107124
}
108125

@@ -117,6 +134,7 @@ class UThreadPrimary : public UThreadBase {
117134
|| secondary_queue_.tryPush(std::move(task)))) {
118135
std::this_thread::yield();
119136
}
137+
cv_.notify_one();
120138
}
121139

122140

@@ -164,16 +182,15 @@ class UThreadPrimary : public UThreadBase {
164182
* 窃取的时候,仅从相邻的primary线程中窃取
165183
* 待窃取相邻的数量,不能超过默认primary线程数
166184
*/
167-
for (int i = 0; i < steal_range_; i++) {
185+
for (auto& target : steal_targets_) {
168186
/**
169187
* 从线程中周围的thread中,窃取任务。
170188
* 如果成功,则返回true,并且执行任务。
171189
* steal 的时候,先从第二个队列里偷,从而降低触碰锁的概率
172190
*/
173-
int curIndex = (index_ + i + 1) % config_->default_thread_size_;
174-
if (likely((*pool_threads_)[curIndex])
175-
&& (((*pool_threads_)[curIndex])->secondary_queue_.trySteal(task))
176-
|| ((*pool_threads_)[curIndex])->primary_queue_.trySteal(task)) {
191+
if (likely((*pool_threads_)[target])
192+
&& (((*pool_threads_)[target])->secondary_queue_.trySteal(task))
193+
|| ((*pool_threads_)[target])->primary_queue_.trySteal(task)) {
177194
return true;
178195
}
179196
}
@@ -192,13 +209,12 @@ class UThreadPrimary : public UThreadBase {
192209
return false;
193210
}
194211

195-
for (int i = 0; i < steal_range_; i++) {
196-
int curIndex = (index_ + i + 1) % config_->default_thread_size_;
197-
if (likely((*pool_threads_)[curIndex])) {
198-
bool result = ((*pool_threads_)[curIndex])->secondary_queue_.trySteal(tasks, config_->max_steal_batch_size_);
212+
for (auto& target : steal_targets_) {
213+
if (likely((*pool_threads_)[target])) {
214+
bool result = ((*pool_threads_)[target])->secondary_queue_.trySteal(tasks, config_->max_steal_batch_size_);
199215
auto leftSize = config_->max_steal_batch_size_ - tasks.size();
200216
if (leftSize > 0) {
201-
result |= ((*pool_threads_)[curIndex])->primary_queue_.trySteal(tasks, leftSize);
217+
result |= ((*pool_threads_)[target])->primary_queue_.trySteal(tasks, leftSize);
202218
}
203219

204220
if (result) {
@@ -216,12 +232,30 @@ class UThreadPrimary : public UThreadBase {
216232
return false;
217233
}
218234

235+
236+
/**
237+
* 构造 steal 范围的 target,避免每次盗取的时候,重复计算
238+
* @return
239+
*/
240+
CVoid buildStealTargets() {
241+
steal_targets_.clear();
242+
for (int i = 0; i < config_->calcStealRange(); i++) {
243+
auto target = (index_ + i + 1) % config_->default_thread_size_;
244+
steal_targets_.push_back(target);
245+
}
246+
steal_targets_.shrink_to_fit();
247+
}
248+
219249
private:
220250
int index_; // 线程index
221-
int steal_range_; // 偷窃的范围信息
222-
UWorkStealingQueue primary_queue_; // 内部队列信息
223-
UWorkStealingQueue secondary_queue_; // 第二个队列,用于减少触锁概率,提升性能
251+
int cur_empty_epoch_ = 0; // 当前空转的轮数信息
252+
UWorkStealingQueue<UTask> primary_queue_; // 内部队列信息
253+
UWorkStealingQueue<UTask> secondary_queue_; // 第二个队列,用于减少触锁概率,提升性能
224254
std::vector<UThreadPrimary *>* pool_threads_; // 用于存放线程池中的线程信息
255+
std::vector<int> steal_targets_; // 被偷的目标信息
256+
257+
std::mutex mutex_;
258+
std::condition_variable cv_;
225259

226260
friend class UThreadPool;
227261
friend class UAllocator;

‎src/UtilsCtrl/ThreadPool/Thread/UThreadSecondary.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class UThreadSecondary : public UThreadBase {
5656
}
5757

5858

59-
CStatus run() override {
59+
CStatus run() final {
6060
CGRAPH_FUNCTION_BEGIN
6161
CGRAPH_ASSERT_INIT(true)
6262

‎src/UtilsCtrl/ThreadPool/UThreadPool.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ CVoid UThreadPool::monitor() {
229229
CGRAPH_SLEEP_SECOND(1)
230230
}
231231

232-
int span = config_.monitor_span_;
232+
auto span = config_.monitor_span_;
233233
while (config_.monitor_enable_ && is_init_ && span--) {
234234
CGRAPH_SLEEP_SECOND(1) // 保证可以快速退出
235235
}

‎src/UtilsCtrl/ThreadPool/UThreadPoolConfig.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ struct UThreadPoolConfig : public CStruct {
2323
int max_local_batch_size_ = CGRAPH_MAX_LOCAL_BATCH_SIZE;
2424
int max_pool_batch_size_ = CGRAPH_MAX_POOL_BATCH_SIZE;
2525
int max_steal_batch_size_ = CGRAPH_MAX_STEAL_BATCH_SIZE;
26+
int primary_thread_busy_epoch_ = CGRAPH_PRIMARY_THREAD_BUSY_EPOCH;
27+
int primary_thread_empty_interval_ = CGRAPH_PRIMARY_THREAD_EMPTY_INTERVAL;
2628
int secondary_thread_ttl_ = CGRAPH_SECONDARY_THREAD_TTL;
27-
int monitor_span_ = CGRAPH_MONITOR_SPAN;
29+
CSec monitor_span_ = CGRAPH_MONITOR_SPAN;
2830
CMSec queue_emtpy_interval_ = CGRAPH_QUEUE_EMPTY_INTERVAL;
2931
int primary_thread_policy_ = CGRAPH_PRIMARY_THREAD_POLICY;
3032
int secondary_thread_policy_ = CGRAPH_SECONDARY_THREAD_POLICY;

‎src/UtilsCtrl/ThreadPool/UThreadPoolDefine.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,19 @@ static const int CGRAPH_LONG_TIME_TASK_STRATEGY = -101;
4949
/**
5050
* 以下为线程池配置信息
5151
*/
52-
static const int CGRAPH_DEFAULT_THREAD_SIZE = 0; // 默认开启主线程个数
53-
static const int CGRAPH_SECONDARY_THREAD_SIZE = 8; // 默认开启辅助线程个数
52+
static const int CGRAPH_DEFAULT_THREAD_SIZE = 8; // 默认开启主线程个数
53+
static const int CGRAPH_SECONDARY_THREAD_SIZE = 0; // 默认开启辅助线程个数
5454
static const int CGRAPH_MAX_THREAD_SIZE = 16; // 最大线程个数
5555
static const int CGRAPH_MAX_TASK_STEAL_RANGE = 2; // 盗取机制相邻范围
5656
static const bool CGRAPH_BATCH_TASK_ENABLE = false; // 是否开启批量任务功能
5757
static const int CGRAPH_MAX_LOCAL_BATCH_SIZE = 2; // 批量执行本地任务最大值
5858
static const int CGRAPH_MAX_POOL_BATCH_SIZE = 2; // 批量执行通用任务最大值
5959
static const int CGRAPH_MAX_STEAL_BATCH_SIZE = 2; // 批量盗取任务最大值
60+
static const int CGRAPH_PRIMARY_THREAD_BUSY_EPOCH = 10; // 主线程进入wait状态的轮数,数值越大,理论性能越高,但空转可能性也越大
61+
static const CMSec CGRAPH_PRIMARY_THREAD_EMPTY_INTERVAL = 3; // 主线程进入休眠状态的默认时间
6062
static const int CGRAPH_SECONDARY_THREAD_TTL = 10; // 辅助线程ttl,单位为s
6163
static const bool CGRAPH_MONITOR_ENABLE = false; // 是否开启监控程序
62-
static const int CGRAPH_MONITOR_SPAN = 5; // 监控线程执行间隔,单位为s
64+
static const CSec CGRAPH_MONITOR_SPAN = 5; // 监控线程执行间隔,单位为s
6365
static const CMSec CGRAPH_QUEUE_EMPTY_INTERVAL = 3; // 队列为空时,等待的时间。仅针对辅助线程,单位为ms
6466
static const bool CGRAPH_BIND_CPU_ENABLE = false; // 是否开启绑定cpu模式(仅针对主线程)
6567
static const int CGRAPH_PRIMARY_THREAD_POLICY = CGRAPH_THREAD_SCHED_OTHER; // 主线程调度策略

‎src/UtilsCtrl/ThreadPool/UThreadPoolInclude.h

+1
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@
1717
#include "Task/UTaskInclude.h"
1818
#include "Thread/UThreadInclude.h"
1919
#include "Lock/ULockInclude.h"
20+
#include "Semaphore/USemaphore.h"
2021

2122
#endif //CGRAPH_UTHREADPOOLINCLUDE_H

0 commit comments

Comments
 (0)
Please sign in to comment.