Apple Network Framework Socket Changes

.github/workflows/ci.yml

@@ -243,7 +243,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_DISPATCH_QUEUE=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_NETWORK_FRAMEWORK=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
 
   macos-x64:
     runs-on: macos-14-large # latest
@@ -274,7 +274,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_DISPATCH_QUEUE=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" --config Debug
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DAWS_USE_APPLE_NETWORK_FRAMEWORK=${{ matrix.eventloop == 'dispatch_queue' && 'ON' || 'OFF' }} --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" --config Debug
 
   freebsd:
     runs-on: ubuntu-24.04  # latest

CMakeLists.txt

@@ -29,6 +29,7 @@ file(GLOB AWS_IO_TESTING_HEADERS
         "include/aws/testing/*.h"
         )
 
+
 file(GLOB AWS_IO_PRIV_HEADERS
         "include/aws/io/private/*.h"
         )

README.md

@@ -151,7 +151,8 @@ Core to Async-IO is the event-loop. We provide an implementation for most platfo
 Platform | Implementation
 --- | ---
 Linux | Edge-Triggered Epoll
-BSD Variants and Apple Devices | KQueue
+BSD Variants | KQueue
+Apple Devices | KQueue or Apple Dispatch Queue
 Windows | IOCP (IO Completion Ports)
 
 Also, you can always implement your own as well.
@@ -645,7 +646,7 @@ All exported functions, simply shim into the v-table and return.
 
 We include a cross-platform API for sockets. We support TCP and UDP using IPv4 and IPv6, and Unix Domain sockets. On Windows,
 we use Named Pipes to support the functionality of Unix Domain sockets. On Windows, this is implemented with winsock2, and on
-all unix platforms we use the posix API.
+all unix platforms we use the posix API. We also provides options to use Apple Network Framework on Apple.
 
 Upon a connection being established, the new socket (either as the result of a `connect()` or `start_accept()` call)
 will not be attached to any event loops. It is your responsibility to register it with an event loop to begin receiving
@@ -715,47 +716,53 @@ upon completion of asynchronous operations. If you are using UDP or LOCAL, `conn
 
 Shuts down any pending operations on the socket, and cleans up state. The socket object can be re initialized after this operation.
 
-    int aws_socket_connect(struct aws_socket *socket, struct aws_socket_endpoint *remote_endpoint);
+    int aws_socket_set_cleanup_complete_callback(struct aws_socket *socket, aws_socket_on_shutdown_complete_fn fn, void *user_data);
 
-Connects to a remote endpoint. In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`,
-and if the operation is successful, the socket can immediately be used for write operations.
+Sets the clean up completion callback. The callback will be invoked if `aws_socket_clean_up()` finish to clean up the socket resources. It is safe to release the socket memory after this callback is invoked.
 
-In TCP, this will function will not block. If the return value is successful, then you must wait on the `on_connection_established()`
-callback to be invoked before using the socket.
+    int aws_socket_connect(struct aws_socket *socket, const struct aws_socket_endpoint *remote_endpoint, struct aws_event_loop *event_loop, aws_socket_on_connection_result_fn *on_connection_result, void *user_data);
+
+Connects to a remote endpoint. In TCP and all Apple Network Framework connections (regardless it is UDP, TCP or LOCAL), when the connection succeed, you still must wait on  the `on_connection_result()` callback to be invoked before using the socket.
+
+In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`, and if the operation is successful, 
+the socket can immediately be used for write operations.
 
 For LOCAL (Unix Domain Sockets or Named Pipes), the socket will be immediately ready for use upon a successful return.
 
     int aws_socket_bind(struct aws_socket *socket, struct aws_socket_endpoint *local_endpoint);
 
-Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented
-modes, you still must call `aws_socket_listen()` and `aws_socket_start_accept()` before using the socket.
+Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented 
+modes or if you are using Apple Network Framework (regardless it is UDP or TCP), you still must call `aws_socket_listen()` and 
+`aws_socket_start_accept()` before using the socket.
 
     int aws_socket_listen(struct aws_socket *socket, int backlog_size);
 
-TCP and LOCAL only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
+TCP, LOCAL, and Apple Network Framework only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
 
-    int aws_socket_start_accept(struct aws_socket *socket);
+    int aws_socket_start_accept(struct aws_socket *socket, struct aws_event_loop *accept_loop, struct aws_socket_listener_options options);
 
-TCP and LOCAL only. The socket will begin accepting new connections. This is an asynchronous operation. New connections will
-arrive via the `on_incoming_connection()` callback.
+TCP, LOCAL, and Apple Network Framework only. The socket will begin accepting new connections. This is an asynchronous operation. `on_accept_start()` will be invoked when the listener is ready to accept new connection. New connections will arrive via the `on_accept_result()` callback. 
 
     int aws_socket_stop_accept(struct aws_socket *socket);
 
-TCP and LOCAL only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` again after this
-operation.
+TCP, LOCAL, and Apple Network Framework only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` 
+again after this operation.
 
     int aws_socket_close(struct aws_socket *socket);
 
 Calls `close()` on the socket and unregisters all io operations from the event loop.
 
+    int aws_socket_set_close_complete_callback(struct aws_socket *socket, aws_socket_on_shutdown_complete_fn fn, void *user_data);
+
+Sets the close completion callback. The callback will be invoked if `aws_socket_close()` finish to process all the I/O events and close the socket.
+
     struct aws_io_handle *aws_socket_get_io_handle(struct aws_socket *socket);
 
 Fetches the underlying io handle for use in event loop registrations and channel handlers.
 
     int aws_socket_set_options(struct aws_socket *socket, struct aws_socket_options *options);
 
-Sets new socket options on the underlying socket. This is mainly useful in context of accepting a new connection via:
-`on_incoming_connection()`.
+Sets new socket options on the underlying socket.
 
     int aws_socket_read(struct aws_socket *socket, struct aws_byte_buf *buffer, size_t *amount_read);
 

include/aws/io/channel_bootstrap.h

@@ -132,6 +132,14 @@ typedef void(aws_server_bootstrap_on_accept_channel_shutdown_fn)(
     struct aws_channel *channel,
     void *user_data);
 
+/**
+ * This function is only used for async listener (Apple Network Framework in this case).
+ * Once the server listener socket is finished setup and starting listening, this fuction
+ * will be invoked.
+ */
+typedef void(
+    aws_server_bootstrap_on_listener_setup_fn)(struct aws_server_bootstrap *bootstrap, int error_code, void *user_data);
+
 /**
  * Once the server listener socket is finished destroying, and all the existing connections are closed, this fuction
  * will be invoked.
@@ -210,6 +218,7 @@ struct aws_server_socket_channel_bootstrap_options {
     uint32_t port;
     const struct aws_socket_options *socket_options;
     const struct aws_tls_connection_options *tls_options;
+    aws_server_bootstrap_on_listener_setup_fn *setup_callback;
     aws_server_bootstrap_on_accept_channel_setup_fn *incoming_callback;
     aws_server_bootstrap_on_accept_channel_shutdown_fn *shutdown_callback;
     aws_server_bootstrap_on_server_listener_destroy_fn *destroy_callback;
@@ -288,6 +297,10 @@ AWS_IO_API int aws_server_bootstrap_set_alpn_callback(
  * shutting down. Immediately after the `shutdown_callback` returns, the channel is cleaned up automatically. All
  * callbacks are invoked the thread of the event-loop that the listening socket is assigned to
  *
+ * `setup_callback`. If set, the callback will be asynchronously invoked when the listener is ready for use. For Apple
+ * Network Framework, the listener is not usable until the callback is invoked. If the listener creation failed
+ * (return NULL), the `setup_callback` will not be invoked.
+ *
  * Upon shutdown of your application, you'll want to call `aws_server_bootstrap_destroy_socket_listener` with the return
  * value from this function.
  *

include/aws/io/private/socket_impl.h

@@ -18,6 +18,20 @@ typedef void (*aws_ms_fn_ptr)(void);
 void aws_check_and_init_winsock(void);
 aws_ms_fn_ptr aws_winsock_get_connectex_fn(void);
 aws_ms_fn_ptr aws_winsock_get_acceptex_fn(void);
+#else // NOT ON WINDOWS
+struct socket_address {
+    union sock_addr_types {
+        struct sockaddr_in addr_in;
+        struct sockaddr_in6 addr_in6;
+        struct sockaddr_un un_addr;
+#    ifdef __APPLE__
+        struct sockaddr addr_base;
+#    endif
+#    ifdef USE_VSOCK
+        struct sockaddr_vm vm_addr;
+#    endif
+    } sock_addr_types;
+};
 #endif
 
 int aws_socket_init_posix(
@@ -48,8 +62,7 @@ struct aws_socket_vtable {
     int (*socket_start_accept_fn)(
         struct aws_socket *socket,
         struct aws_event_loop *accept_loop,
-        aws_socket_on_accept_result_fn *on_accept_result,
-        void *user_data);
+        struct aws_socket_listener_options options);
     int (*socket_stop_accept_fn)(struct aws_socket *socket);
     int (*socket_close_fn)(struct aws_socket *socket);
     int (*socket_shutdown_dir_fn)(struct aws_socket *socket, enum aws_channel_direction dir);
@@ -67,6 +80,20 @@ struct aws_socket_vtable {
         void *user_data);
     int (*socket_get_error_fn)(struct aws_socket *socket);
     bool (*socket_is_open_fn)(struct aws_socket *socket);
+    int (*socket_set_close_callback)(struct aws_socket *socket, aws_socket_on_shutdown_complete_fn fn, void *user_data);
+    int (*socket_set_cleanup_callback)(
+        struct aws_socket *socket,
+        aws_socket_on_shutdown_complete_fn fn,
+        void *user_data);
+};
+
+struct on_start_accept_result_args {
+    struct aws_task task;
+    int error;
+    struct aws_allocator *allocator;
+    struct aws_socket *socket;
+    aws_socket_on_accept_started_fn *on_accept_start;
+    void *on_accept_start_user_data;
 };
 
 #endif // AWS_IO_SOCKET_IMPL_H

include/aws/io/socket.h

@@ -50,6 +50,8 @@ enum aws_socket_impl_type {
 
 #define AWS_NETWORK_INTERFACE_NAME_MAX 16
 
+typedef void(aws_socket_on_shutdown_complete_fn)(void *user_data);
+
 struct aws_socket_options {
     enum aws_socket_type type;
     enum aws_socket_domain domain;
@@ -90,6 +92,15 @@ struct aws_event_loop;
  */
 typedef void(aws_socket_on_connection_result_fn)(struct aws_socket *socket, int error_code, void *user_data);
 
+/**
+ * Called by a listening socket when a listener accept has successfully initialized or an error has occurred.
+ * If the listener was successful error_code will be AWS_ERROR_SUCCESS and the socket has already been assigned
+ * to the event loop specified in aws_socket_start_accept().
+ *
+ * If an error occurred error_code will be non-zero.
+ */
+typedef void(aws_socket_on_accept_started_fn)(struct aws_socket *socket, int error_code, void *user_data);
+
 /**
  * Called by a listening socket when either an incoming connection has been received or an error occurred.
  *
@@ -116,14 +127,15 @@ typedef void(aws_socket_on_accept_result_fn)(
  * Callback for when the data passed to a call to aws_socket_write() has either completed or failed.
  * On success, error_code will be AWS_ERROR_SUCCESS.
  *
- * `socket` may be NULL in the callback if the socket is released and cleaned up before a callback is triggered.
- * by the system I/O handler,
+ * `socket` may be NULL in the callback if the socket is released and cleaned up before the callback is triggered.
  */
 typedef void(
     aws_socket_on_write_completed_fn)(struct aws_socket *socket, int error_code, size_t bytes_written, void *user_data);
 /**
  * Callback for when socket is either readable (edge-triggered) or when an error has occurred. If the socket is
  * readable, error_code will be AWS_ERROR_SUCCESS.
+ *
+ * `socket` may be NULL in the callback if the socket is released and cleaned up before the callback is triggered.
  */
 typedef void(aws_socket_on_readable_fn)(struct aws_socket *socket, int error_code, void *user_data);
 
@@ -156,6 +168,16 @@ struct aws_socket {
     void *impl;
 };
 
+struct aws_socket_listener_options {
+    aws_socket_on_accept_result_fn *on_accept_result;
+    void *on_accept_result_user_data;
+
+    // This callback is invoked when the listener starts accepting incoming connections.
+    // If the callback set, the socket must not be released before the callback invoked.
+    aws_socket_on_accept_started_fn *on_accept_start;
+    void *on_accept_start_user_data;
+};
+
 struct aws_byte_buf;
 struct aws_byte_cursor;
 
@@ -227,8 +249,7 @@ AWS_IO_API int aws_socket_listen(struct aws_socket *socket, int backlog_size);
 AWS_IO_API int aws_socket_start_accept(
     struct aws_socket *socket,
     struct aws_event_loop *accept_loop,
-    aws_socket_on_accept_result_fn *on_accept_result,
-    void *user_data);
+    struct aws_socket_listener_options options);
 
 /**
  * TCP, LOCAL and VSOCK only. The listening socket will stop accepting new connections.
@@ -245,6 +266,9 @@ AWS_IO_API int aws_socket_stop_accept(struct aws_socket *socket);
  * non-event-loop thread or the event-loop the socket is currently assigned to. If called from outside the event-loop,
  * this function will block waiting on the socket to close. If this is called from an event-loop thread other than
  * the one it's assigned to, it presents the possibility of a deadlock, so don't do it.
+ *
+ * If you are using Apple Network Framework, you should always call this function from an event-loop thread regardless
+ * it is a server or client socket.
  */
 AWS_IO_API int aws_socket_close(struct aws_socket *socket);
 
@@ -254,8 +278,7 @@ AWS_IO_API int aws_socket_close(struct aws_socket *socket);
 AWS_IO_API int aws_socket_shutdown_dir(struct aws_socket *socket, enum aws_channel_direction dir);
 
 /**
- * Sets new socket options on the underlying socket. This is mainly useful in context of accepting a new connection via:
- * `on_incoming_connection()`. options is copied.
+ * Sets new socket options on the underlying socket.
  */
 AWS_IO_API int aws_socket_set_options(struct aws_socket *socket, const struct aws_socket_options *options);
 
@@ -319,6 +342,24 @@ AWS_IO_API int aws_socket_write(
     aws_socket_on_write_completed_fn *written_fn,
     void *user_data);
 
+/**
+ * Apple Network Framework only. The callback that will triggered when aws_socket_close() finished. The callback
+ * will be called from the socket event loop.
+ */
+AWS_IO_API int aws_socket_set_close_complete_callback(
+    struct aws_socket *socket,
+    aws_socket_on_shutdown_complete_fn fn,
+    void *user_data);
+
+/**
+ * Apple Network Framework only. The callback that will triggered when aws_socket_cleanup() finished. And
+ * it is only safe to release the socket afterwards. The callback will be called from the socket event loop.
+ */
+AWS_IO_API int aws_socket_set_cleanup_complete_callback(
+    struct aws_socket *socket,
+    aws_socket_on_shutdown_complete_fn fn,
+    void *user_data);
+
 /**
  * Gets the latest error from the socket. If no error has occurred AWS_OP_SUCCESS will be returned. This function does
  * not raise any errors to the installed error handlers.
@@ -358,6 +399,12 @@ AWS_IO_API void aws_socket_endpoint_init_local_address_for_test(struct aws_socke
  * network_interface_name on Windows */
 AWS_IO_API bool aws_is_network_interface_name_valid(const char *interface_name);
 
+/**
+ * Get default impl type based on the platform.
+ * For user in internal tests only.
+ */
+AWS_IO_API enum aws_socket_impl_type aws_socket_get_default_impl_type(void);
+
 AWS_EXTERN_C_END
 AWS_POP_SANE_WARNING_LEVEL