Opt-in busywait mode for futexes

Author: kateinoigakukun

Makefile

@@ -369,6 +369,7 @@ LIBC_TOP_HALF_MUSL_SOURCES += \
         thread/sem_trywait.c \
         thread/sem_wait.c \
         thread/wasm32/wasi_thread_start.s \
+        thread/wasm32/__wasilibc_busywait.c \
     )
 endif
 ifeq ($(THREAD_MODEL), single)

expected/wasm32-wasip1-threads/defined-symbols.txt

@@ -362,13 +362,16 @@ __wasilibc_cwd_lock
 __wasilibc_cwd_unlock
 __wasilibc_deinitialize_environ
 __wasilibc_dttoif
+__wasilibc_enable_futex_busywait_on_current_thread
 __wasilibc_ensure_environ
 __wasilibc_environ
 __wasilibc_fd_renumber
 __wasilibc_find_abspath
 __wasilibc_find_relpath
 __wasilibc_find_relpath_alloc
 __wasilibc_futex_wait
+__wasilibc_futex_wait_atomic_wait
+__wasilibc_futex_wait_maybe_busy
 __wasilibc_get_environ
 __wasilibc_iftodt
 __wasilibc_initialize_environ
@@ -400,6 +403,7 @@ __wasilibc_rmdirat
 __wasilibc_stat
 __wasilibc_tell
 __wasilibc_unlinkat
+__wasilibc_use_busy_futex
 __wasilibc_utimens
 __wasm_call_dtors
 __wcscoll_l

expected/wasm32-wasip1-threads/include-all.c

@@ -167,6 +167,7 @@
 #include <utime.h>
 #include <values.h>
 #include <wasi/api.h>
+#include <wasi/libc-busywait.h>
 #include <wasi/libc-environ.h>
 #include <wasi/libc-find-relpath.h>
 #include <wasi/libc-nocwd.h>

expected/wasm32-wasip1-threads/predefined-macros.txt

@@ -3091,6 +3091,7 @@
 #define __va_copy(d,s) __builtin_va_copy(d,s)
 #define __wasi__ 1
 #define __wasi_api_h 
+#define __wasi_libc_busywait_h
 #define __wasi_libc_environ_h 
 #define __wasi_libc_find_relpath_h 
 #define __wasi_libc_h 

libc-bottom-half/headers/public/wasi/libc.h

@@ -64,6 +64,9 @@ int __wasilibc_rename_oldat(int olddirfd, const char *oldpath, const char *newpa
 int __wasilibc_rename_newat(const char *oldpath, int newdirfd, const char *newpath)
     __attribute__((__warn_unused_result__));
 
+/// Enable busywait in futex on current thread.
+void __wasilibc_enable_futex_busywait_on_current_thread(void);
+
 #ifdef __cplusplus
 }
 #endif

libc-top-half/musl/include/wasi/libc-busywait.h

@@ -0,0 +1,15 @@
+#ifndef __wasi_libc_busywait_h
+#define __wasi_libc_busywait_h
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// Enable busywait in futex on current thread.
+void __wasilibc_enable_futex_busywait_on_current_thread(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif 

libc-top-half/musl/src/thread/__wait.c

@@ -4,19 +4,18 @@
 #endif
 
 #ifndef __wasilibc_unmodified_upstream
+
+weak int __wasilibc_futex_wait_maybe_busy(volatile void *addr, int op, int val, int64_t max_wait_ns);
+
 // Use WebAssembly's `wait` instruction to implement a futex. Note that `op` is
 // unused but retained as a parameter to match the original signature of the
 // syscall and that, for `max_wait_ns`, -1 (or any negative number) means wait
 // indefinitely.
 //
 // Adapted from Emscripten: see
 // https://github.com/emscripten-core/emscripten/blob/058a9fff/system/lib/pthread/emscripten_futex_wait.c#L111-L150.
-int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait_ns)
+int __wasilibc_futex_wait_atomic_wait(volatile void *addr, int op, int val, int64_t max_wait_ns)
 {
-    if ((((intptr_t)addr) & 3) != 0) {
-        return -EINVAL;
-    }
-
     int ret = __builtin_wasm_memory_atomic_wait32((int *)addr, val, max_wait_ns);
 
     // memory.atomic.wait32 returns:
@@ -32,6 +31,18 @@ int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait
     assert(ret == 0);
     return 0;
 }
+
+int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait_ns)
+{
+    if ((((intptr_t)addr) & 3) != 0) {
+        return -EINVAL;
+    }
+
+    if (__wasilibc_futex_wait_maybe_busy) {
+        return __wasilibc_futex_wait_maybe_busy(addr, op, val, max_wait_ns);
+    }
+    return __wasilibc_futex_wait_atomic_wait(addr, op, val, max_wait_ns);
+}
 #endif
 
 void __wait(volatile int *addr, volatile int *waiters, int val, int priv)

libc-top-half/musl/src/thread/wasm32/__wasilibc_busywait.c

@@ -0,0 +1,75 @@
+#include <stdint.h>
+#include <time.h>
+#include <errno.h>
+
+#define DEFINE_GLOBAL_GETTER(name, core_type, c_type) \
+    static inline c_type name##_get(void) { \
+        c_type val; \
+        __asm__( \
+            ".globaltype " #name ", " #core_type "\n" \
+            "global.get " #name "\n" \
+            "local.set %0\n" \
+            : "=r"(val)); \
+        return val; \
+    }
+#define DEFINE_GLOBAL_SETTER(name, core_type, c_type) \
+    static inline void name##_set(c_type val) { \
+        __asm__( \
+            ".globaltype " #name ", " #core_type "\n" \
+            "local.get %0\n" \
+            "global.set " #name "\n" \
+            : : "r"(val)); \
+    }
+
+#define DEFINE_RW_GLOBAL(name, core_type, c_type) \
+    __asm__( \
+        ".globaltype " #name ", " #core_type "\n" \
+        ".global " #name "\n" \
+        #name ":\n" \
+    ); \
+    DEFINE_GLOBAL_GETTER(name, core_type, c_type) \
+    DEFINE_GLOBAL_SETTER(name, core_type, c_type)
+
+DEFINE_RW_GLOBAL(__wasilibc_use_busy_futex, i32, int32_t)
+
+void __wasilibc_enable_futex_busywait_on_current_thread(void)
+{
+    __wasilibc_use_busy_futex_set(1);
+}
+
+int __wasilibc_futex_wait_atomic_wait(volatile void *addr, int op, int val, int64_t max_wait_ns);
+
+int __wasilibc_futex_wait_maybe_busy(volatile void *addr, int op, int val, int64_t max_wait_ns)
+{
+    // PLEASE NOTE THAT WE CANNOT CALL LIBC FUNCTIONS THAT USE FUTEXES HERE
+
+    if (!__wasilibc_use_busy_futex_get()) {
+        return __wasilibc_futex_wait_atomic_wait(addr, op, val, max_wait_ns);
+    }
+
+    struct timespec start;
+    int r = clock_gettime(CLOCK_REALTIME, &start);
+
+    // If we can't get the current time, we can't wait with a timeout.
+    if (r) return r;
+
+    while (1) {
+        // Check timeout if it's a positive value
+        if (max_wait_ns >= 0) {
+            struct timespec now;
+            r = clock_gettime(CLOCK_REALTIME, &now);
+            if (r) return r;
+
+            int64_t elapsed_ns = (now.tv_sec - start.tv_sec) * 1000000000 + now.tv_nsec - start.tv_nsec;
+            if (elapsed_ns >= max_wait_ns) {
+                return -ETIMEDOUT;
+            }
+        }
+
+        if (__c11_atomic_load((_Atomic int *)addr, __ATOMIC_SEQ_CST) != val) {
+            break;
+        }
+    }
+
+    return 0;
+} 

test/src/libc-test/functional/pthread_cond_busywait.c

@@ -0,0 +1,13 @@
+//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads
+//! add-flags.py(CFLAGS): -I.
+//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824
+//! add-flags.py(RUN): --wasi threads
+#include "build/download/libc-test/src/functional/pthread_cond.c"
+
+#include <wasi/libc-busywait.h>
+
+__attribute__((constructor))
+void __wasilibc_enable_busywait(void)
+{
+    __wasilibc_enable_futex_busywait_on_current_thread();
+}

test/src/libc-test/functional/pthread_mutex_busywait.c

@@ -0,0 +1,13 @@
+//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads
+//! add-flags.py(CFLAGS): -I.
+//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824
+//! add-flags.py(RUN): --wasi threads
+#include "build/download/libc-test/src/functional/pthread_mutex.c"
+
+#include <wasi/libc-busywait.h>
+
+__attribute__((constructor))
+void __wasilibc_enable_busywait(void)
+{
+    __wasilibc_enable_futex_busywait_on_current_thread();
+}

test/src/libc-test/functional/pthread_tsd_busywait.c

@@ -0,0 +1,13 @@
+//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads
+//! add-flags.py(CFLAGS): -I.
+//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824
+//! add-flags.py(RUN): --wasi threads
+#include "build/download/libc-test/src/functional/pthread_tsd.c"
+
+#include <wasi/libc-busywait.h>
+
+__attribute__((constructor))
+void __wasilibc_enable_busywait(void)
+{
+    __wasilibc_enable_futex_busywait_on_current_thread();
+}

test/src/misc/busywait.c

@@ -0,0 +1,32 @@
+//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads
+//! add-flags.py(CFLAGS): -pthread
+//! add-flags.py(LDFLAGS): -pthread
+//! add-flags.py(RUN): --wasi threads
+
+#include <assert.h>
+#include <pthread.h>
+#include <time.h>
+#include <errno.h>
+#include <wasi/libc-busywait.h>
+
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+
+int main() {
+    struct timespec ts;
+    int ret;
+
+    clock_gettime(CLOCK_REALTIME, &ts);
+    ts.tv_sec += 1;
+
+    __wasilibc_enable_futex_busywait_on_current_thread();
+
+    pthread_mutex_lock(&mutex);
+
+    ret = pthread_cond_timedwait(&cond, &mutex, &ts);
+
+    assert(ret == ETIMEDOUT);
+
+    pthread_mutex_unlock(&mutex);
+    return 0;
+}