Support hosted builds against Arduino-Emulator (#420)

MitchBradley · mathieucarbou · pre-commit-ci-lite[bot] · web-flow · commit c25aabc64530 · 2026-04-08T18:51:04.000+02:00
* Support hosted builds against Arduino-Emulator * Factors the locking into one place * Adds "|| defined(HOST)" as needed elsewhere * Adds "override" in a few places where it was missing * Changed comments to address Copilot nitpicking * Changed cpp macros to c++ possibly-null classes, thanks willmmiles * locks in asyncsrv namespace, guarded by ASYNCSRV_USE_MUTEX * HOST versions of logging. * __unused -> __asyncws_unused * More explicit formulation of ASYNCWEBSERVER_USE_MUTEX define * ci(pre-commit): Apply automatic fixes * Apply suggestion from @mathieucarbou * Fix compile sisue * Add CI for Arduino-Emulator * Fix compilation issue where tcp_state enum type was not found project must be compiled with lwip when using arduino emulator * Add CI for Arduino-Emulator * ci(pre-commit): Apply automatic fixes * Fix Arduino Elumator CI * Move FPSTR in cpp * Try fix CI on linux (works on mac) * Project Structure Reorg: regrouping all examples under examples folder * Move arduino_emulator to examples folder --------- Co-authored-by: Mathieu Carbou <mathieu.carbou@gmail.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
diff --git a/.github/workflows/build-arduino-emulator.yml b/.github/workflows/build-arduino-emulator.yml
@@ -0,0 +1,52 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
+name: Build (Arduino Emulator)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - release/*
+    paths-ignore:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "README.md"
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "README.md"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  arduino-emulator:
+    name: Arduino Emulator (cmake, HOST)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install build tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake ninja-build build-essential git
+
+      - name: Clone Arduino-Emulator
+        run: |
+          git clone --depth 1 --recurse-submodules https://github.com/pschatzmann/Arduino-Emulator.git .ci/arduino-emulator
+
+      - name: Clone AsyncTCP, Arduino FS headers, lwIP and lwIP contrib
+        run: |
+          git clone --depth 1 https://github.com/ESP32Async/AsyncTCP .ci/asynctcp
+          git clone --depth 1 https://github.com/espressif/arduino-esp32.git .ci/arduino-esp32
+          git clone --depth 1 https://github.com/lwip-tcpip/lwip.git .ci/lwip
+          git clone --depth 1 https://git.savannah.nongnu.org/git/lwip/lwip-contrib.git .ci/lwip-contrib
+
+      - name: Build with Arduino-Emulator
+        run: |
+          cmake -S examples/arduino_emulator -B .ci/arduino-emulator-build/out -G Ninja
+          cmake --build .ci/arduino-emulator-build/out --target espasyncwebserver --parallel
diff --git a/examples/arduino/CaptivePortal/CaptivePortal.ino b/examples/arduino/CaptivePortal/CaptivePortal.ino
@@ -19,7 +19,7 @@ static AsyncWebServer server(80);
 
 class CaptiveRequestHandler : public AsyncWebHandler {
 public:
-  bool canHandle(__unused AsyncWebServerRequest *request) const override {
+  bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {
     return true;
   }
 
diff --git a/examples/arduino/Filters/Filters.ino b/examples/arduino/Filters/Filters.ino
@@ -23,7 +23,7 @@ static AsyncWebServer server(80);
 
 class CaptiveRequestHandler : public AsyncWebHandler {
 public:
-  bool canHandle(__unused AsyncWebServerRequest *request) const override {
+  bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {
     return true;
   }
 
diff --git a/examples/arduino/HTTPMethodsWithArduino/HTTPMethodsWithArduino.ino b/examples/arduino/HTTPMethodsWithArduino/HTTPMethodsWithArduino.ino
@@ -58,7 +58,7 @@ static WebRequestMethodComposite allowed = AsyncWebRequestMethod::HTTP_HEAD | As
 
 class MyRequestHandler : public AsyncWebHandler {
 public:
-  bool canHandle(__unused AsyncWebServerRequest *request) const override {
+  bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {
     // Test backward compatibility with previous way of checking if a method is allowed in a composite using a bit operator
     return allowed & request->method();
   }
diff --git a/examples/arduino/HTTPMethodsWithESPIDF/HTTPMethodsWithESPIDF.ino b/examples/arduino/HTTPMethodsWithESPIDF/HTTPMethodsWithESPIDF.ino
@@ -57,7 +57,7 @@ static WebRequestMethodComposite allowed = AsyncWebRequestMethod::HTTP_HEAD | As
 
 class MyRequestHandler : public AsyncWebHandler {
 public:
-  bool canHandle(__unused AsyncWebServerRequest *request) const override {
+  bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {
     // Test backward compatibility with previous way of checking if a method is allowed in a composite using a bit operator
     return allowed & request->method();
   }
diff --git a/examples/arduino_emulator/CMakeLists.txt b/examples/arduino_emulator/CMakeLists.txt
@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.11)
+project(espasyncwebserver_host_compile LANGUAGES C CXX)
+
+add_subdirectory(${CMAKE_SOURCE_DIR}/../../.ci/arduino-emulator ${CMAKE_BINARY_DIR}/arduino-emulator)
+file(GLOB WEB_SRC "${CMAKE_SOURCE_DIR}/../../src/*.cpp")
+add_library(espasyncwebserver STATIC ${WEB_SRC})
+
+target_compile_definitions(espasyncwebserver PUBLIC HOST ARDUINO=10813)
+target_include_directories(espasyncwebserver PUBLIC
+  ${CMAKE_SOURCE_DIR}/../../src
+  ${CMAKE_SOURCE_DIR}/../../.ci/asynctcp/src
+  ${CMAKE_SOURCE_DIR}/../../.ci/arduino-esp32/libraries/FS/src
+  ${CMAKE_SOURCE_DIR}/../../.ci/lwip/src/include
+  ${CMAKE_SOURCE_DIR}/../../.ci/lwip-contrib/ports/unix/port/include
+  ${CMAKE_SOURCE_DIR}/host_config
+)
+target_link_libraries(espasyncwebserver PUBLIC arduino_emulator)
diff --git a/examples/arduino_emulator/host_config/freertos/semphr.h b/examples/arduino_emulator/host_config/freertos/semphr.h
@@ -0,0 +1,3 @@
+#pragma once
+
+typedef void *SemaphoreHandle_t;
diff --git a/examples/arduino_emulator/host_config/lwipopts.h b/examples/arduino_emulator/host_config/lwipopts.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#define NO_SYS       1
+#define LWIP_SOCKET  0
+#define LWIP_NETCONN 0
+#define LWIP_TCP     1
+#define LWIP_IPV6    1
+
+#if defined(__linux__) && !defined(LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS)
+#define LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS 1
+#endif
diff --git a/examples/arduino_emulator/host_config/sdkconfig.h b/examples/arduino_emulator/host_config/sdkconfig.h
@@ -0,0 +1,3 @@
+#pragma once
+
+#define LWIP_IPV6 1
diff --git a/src/AsyncEventSource.cpp b/src/AsyncEventSource.cpp
@@ -191,19 +191,15 @@ AsyncEventSourceClient::AsyncEventSourceClient(AsyncWebServerRequest *request, A
 }
 
 AsyncEventSourceClient::~AsyncEventSourceClient() {
-#ifdef ESP32
   // Protect message queue access (size checks and modifications) which is not thread-safe.
-  std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+  asyncsrv::lock_guard_type lock(_lockmq);
   _messageQueue.clear();
   close();
 }
 
 bool AsyncEventSourceClient::_queueMessage(const char *message, size_t len) {
-#ifdef ESP32
   // Protect message queue access (size checks and modifications) which is not thread-safe.
-  std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+  asyncsrv::lock_guard_type lock(_lockmq);
 
   if (_messageQueue.size() >= SSE_MAX_QUEUED_MESSAGES) {
     async_ws_log_w("Event message queue overflow: discard message");
@@ -231,10 +227,8 @@ bool AsyncEventSourceClient::_queueMessage(const char *message, size_t len) {
 }
 
 bool AsyncEventSourceClient::_queueMessage(AsyncEvent_SharedData_t &&msg) {
-#ifdef ESP32
   // Protect message queue access (size checks and modifications) which is not thread-safe.
-  std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+  asyncsrv::lock_guard_type lock(_lockmq);
 
   if (_messageQueue.size() >= SSE_MAX_QUEUED_MESSAGES) {
     async_ws_log_w("Event message queue overflow: discard message");
@@ -261,10 +255,8 @@ bool AsyncEventSourceClient::_queueMessage(AsyncEvent_SharedData_t &&msg) {
 }
 
 void AsyncEventSourceClient::_onAck(size_t len __attribute__((unused)), uint32_t time __attribute__((unused))) {
-#ifdef ESP32
   // Protect message queue access (size checks and modifications) which is not thread-safe.
-  std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+  asyncsrv::lock_guard_type lock(_lockmq);
 
   // adjust in-flight len
   if (len < _inflight) {
@@ -289,10 +281,8 @@ void AsyncEventSourceClient::_onAck(size_t len __attribute__((unused)), uint32_t
 }
 
 void AsyncEventSourceClient::_onPoll() {
-#ifdef ESP32
   // Protect message queue access (size checks and modifications) which is not thread-safe.
-  std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+  asyncsrv::lock_guard_type lock(_lockmq);
   if (_messageQueue.size()) {
     _runQueue();
   }
@@ -373,10 +363,7 @@ void AsyncEventSource::_addClient(AsyncEventSourceClient *client) {
     _connectcb(client);
   }
 
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
-
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   _clients.emplace_back(client);
 
   _adjust_inflight_window();
@@ -386,9 +373,7 @@ void AsyncEventSource::_handleDisconnect(AsyncEventSourceClient *client) {
   if (_disconnectcb) {
     _disconnectcb(client);
   }
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   for (auto i = _clients.begin(); i != _clients.end(); ++i) {
     if (i->get() == client) {
       _clients.erase(i);
@@ -402,9 +387,7 @@ void AsyncEventSource::close() {
   // While the whole loop is not done, the linked list is locked and so the
   // iterator should remain valid even when AsyncEventSource::_handleDisconnect()
   // is called very early
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   for (const auto &c : _clients) {
     if (c->connected()) {
       /**
@@ -421,9 +404,7 @@ void AsyncEventSource::close() {
 size_t AsyncEventSource::avgPacketsWaiting() const {
   size_t aql = 0;
   uint32_t nConnectedClients = 0;
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   for (const auto &c : _clients) {
     if (c->connected()) {
       aql += c->packetsWaiting();
@@ -435,9 +416,7 @@ size_t AsyncEventSource::avgPacketsWaiting() const {
 
 AsyncEventSource::SendStatus AsyncEventSource::send(const char *message, const char *event, uint32_t id, uint32_t reconnect) {
   AsyncEvent_SharedData_t shared_msg = std::make_shared<String>(generateEventMessage(message, event, id, reconnect));
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   size_t hits = 0;
   size_t miss = 0;
   for (const auto &c : _clients) {
@@ -453,9 +432,7 @@ AsyncEventSource::SendStatus AsyncEventSource::send(const char *message, const c
 }
 
 size_t AsyncEventSource::count() const {
-#ifdef ESP32
-  std::lock_guard<std::recursive_mutex> lock(_client_queue_lock);
-#endif
+  asyncsrv::lock_guard_type lock(_client_queue_lock);
   size_t n_clients{0};
   for (const auto &i : _clients) {
     if (i->connected()) {
diff --git a/src/AsyncEventSource.h b/src/AsyncEventSource.h
@@ -5,7 +5,7 @@
 
 #include <Arduino.h>
 
-#if defined(ESP32) || defined(LIBRETINY)
+#if defined(ESP32) || defined(LIBRETINY) || defined(HOST)
 #include <AsyncTCP.h>
 #ifdef LIBRETINY
 #ifdef round
@@ -136,9 +136,7 @@ class AsyncEventSourceClient {
   size_t _inflight{0};                    // num of unacknowledged bytes that has been written to socket buffer
   size_t _max_inflight{SSE_MAX_INFLIGH};  // max num of unacknowledged bytes that could be written to socket buffer
   std::list<AsyncEventSourceMessage> _messageQueue;
-#ifdef ESP32
-  mutable std::recursive_mutex _lockmq;
-#endif
+  mutable asyncsrv::mutex_type _lockmq;
   bool _queueMessage(const char *message, size_t len);
   bool _queueMessage(AsyncEvent_SharedData_t &&msg);
   void _runQueue();
@@ -205,9 +203,7 @@ class AsyncEventSourceClient {
     return _lastId;
   }
   size_t packetsWaiting() const {
-#ifdef ESP32
-    std::lock_guard<std::recursive_mutex> lock(_lockmq);
-#endif
+    asyncsrv::lock_guard_type lock(_lockmq);
     return _messageQueue.size();
   };
 
@@ -245,11 +241,9 @@ class AsyncEventSource : public AsyncWebHandler {
 private:
   String _url;
   std::list<std::unique_ptr<AsyncEventSourceClient>> _clients;
-#ifdef ESP32
   // Same as for individual messages, protect mutations of _clients list
   // since simultaneous access from different tasks is possible
-  mutable std::recursive_mutex _client_queue_lock;
-#endif
+  mutable asyncsrv::mutex_type _client_queue_lock;
   ArEventHandlerFunction _connectcb = nullptr;
   ArEventHandlerFunction _disconnectcb = nullptr;
 
@@ -331,11 +325,11 @@ class AsyncEventSourceResponse : public AsyncWebServerResponse {
 
 public:
   AsyncEventSourceResponse(AsyncEventSource *server);
-  void _respond(AsyncWebServerRequest *request);
+  void _respond(AsyncWebServerRequest *request) override;
   size_t _ack(AsyncWebServerRequest *request, size_t len, uint32_t time) override {
     return 0;
   };
-  bool _sourceValid() const {
+  bool _sourceValid() const override {
     return true;
   }
 };
diff --git a/src/AsyncJson.h b/src/AsyncJson.h
@@ -118,8 +118,8 @@ class AsyncCallbackJsonWebHandler : public AsyncWebHandler {
   bool canHandle(AsyncWebServerRequest *request) const final;
   void handleRequest(AsyncWebServerRequest *request) final;
   void handleUpload(
-    __unused AsyncWebServerRequest *request, __unused const String &filename, __unused size_t index, __unused uint8_t *data, __unused size_t len,
-    __unused bool final
+    __asyncws_unused AsyncWebServerRequest *request, __asyncws_unused const String &filename, __asyncws_unused size_t index, __asyncws_unused uint8_t *data,
+    __asyncws_unused size_t len, __asyncws_unused bool final
   ) final {}
   void handleBody(AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total) final;
   bool isRequestHandlerTrivial() const final {
diff --git a/src/AsyncWebServerLogging.h b/src/AsyncWebServerLogging.h
@@ -31,6 +31,64 @@
 #define async_ws_log_d(format, ...) log_d(format, ##__VA_ARGS__)
 #define async_ws_log_v(format, ...) log_v(format, ##__VA_ARGS__)
 
+#elif defined(HOST)
+#include <cstdio>
+// Arduino-Emulator has Serial, but it does not have Serial.printf, which
+// is not a member of the Arduino Core API Print class.
+
+// define log levels
+#define ASYNC_WS_LOG_NONE    0 /*!< No log output */
+#define ASYNC_WS_LOG_ERROR   1 /*!< Critical errors, software module can not recover on its own */
+#define ASYNC_WS_LOG_WARN    2 /*!< Error conditions from which recovery measures have been taken */
+#define ASYNC_WS_LOG_INFO    3 /*!< Information messages which describe normal flow of events */
+#define ASYNC_WS_LOG_DEBUG   4 /*!< Extra information which is not necessary for normal use (values, pointers, sizes, etc). */
+#define ASYNC_WS_LOG_VERBOSE 5 /*!< Verbose information for debugging purposes */
+#define ASYNC_WS_LOG_MAX     6 /*!< Number of levels supported */
+// set default log level
+#ifndef ASYNCWEBSERVER_LOG_LEVEL
+#define ASYNCWEBSERVER_LOG_LEVEL ASYNC_WS_LOG_INFO
+#endif
+// error
+#if ASYNCWEBSERVER_LOG_LEVEL >= ASYNC_WS_LOG_ERROR
+#define async_ws_log_e(format, ...)                                               \
+  ::printf("E async_ws %s() %d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+  ::putchar('\n');
+#else
+#define async_ws_log_e(format, ...)
+#endif
+// warn
+#if ASYNCWEBSERVER_LOG_LEVEL >= ASYNC_WS_LOG_WARN
+#define async_ws_log_w(format, ...)                                               \
+  ::printf("W async_ws %s() %d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+  ::putchar('\n');
+#else
+#define async_ws_log_w(format, ...)
+#endif
+// info
+#if ASYNCWEBSERVER_LOG_LEVEL >= ASYNC_WS_LOG_INFO
+#define async_ws_log_i(format, ...)                                               \
+  ::printf("I async_ws %s() %d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+  ::putchar('\n');
+#else
+#define async_ws_log_i(format, ...)
+#endif
+// debug
+#if ASYNCWEBSERVER_LOG_LEVEL >= ASYNC_WS_LOG_DEBUG
+#define async_ws_log_d(format, ...)                                               \
+  ::printf("D async_ws %s() %d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+  ::putchar('\n');
+#else
+#define async_ws_log_d(format, ...)
+#endif
+// verbose
+#if ASYNCWEBSERVER_LOG_LEVEL >= ASYNC_WS_LOG_VERBOSE
+#define async_ws_log_v(format, ...)                                               \
+  ::printf("V async_ws %s() %d: " format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
+  ::putchar('\n');
+#else
+#define async_ws_log_v(format, ...)
+#endif
+
 /**
  * Raspberry Pi Pico specific configurations
  */
diff --git a/src/AsyncWebSocket.cpp b/src/AsyncWebSocket.cpp
diff --git a/src/AsyncWebSocket.h b/src/AsyncWebSocket.h
diff --git a/src/ESPAsyncWebServer.h b/src/ESPAsyncWebServer.h
diff --git a/src/WebAuthentication.cpp b/src/WebAuthentication.cpp
diff --git a/src/WebRequest.cpp b/src/WebRequest.cpp
diff --git a/src/WebServer.cpp b/src/WebServer.cpp

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ static AsyncWebServer server(80);`
`19`	`19`
`20`	`20`	`class CaptiveRequestHandler : public AsyncWebHandler {`
`21`	`21`	`public:`
`22`		`- bool canHandle(__unused AsyncWebServerRequest *request) const override {`
	`22`	`+ bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {`
`23`	`23`	`return true;`
`24`	`24`	`}`
`25`	`25`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ static AsyncWebServer server(80);`
`23`	`23`
`24`	`24`	`class CaptiveRequestHandler : public AsyncWebHandler {`
`25`	`25`	`public:`
`26`		`- bool canHandle(__unused AsyncWebServerRequest *request) const override {`
	`26`	`+ bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {`
`27`	`27`	`return true;`
`28`	`28`	`}`
`29`	`29`
Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,7 @@ static WebRequestMethodComposite allowed = AsyncWebRequestMethod::HTTP_HEAD \| As`
`58`	`58`
`59`	`59`	`class MyRequestHandler : public AsyncWebHandler {`
`60`	`60`	`public:`
`61`		`- bool canHandle(__unused AsyncWebServerRequest *request) const override {`
	`61`	`+ bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {`
`62`	`62`	`// Test backward compatibility with previous way of checking if a method is allowed in a composite using a bit operator`
`63`	`63`	`return allowed & request->method();`
`64`	`64`	`}`
Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,7 @@ static WebRequestMethodComposite allowed = AsyncWebRequestMethod::HTTP_HEAD \| As`
`57`	`57`
`58`	`58`	`class MyRequestHandler : public AsyncWebHandler {`
`59`	`59`	`public:`
`60`		`- bool canHandle(__unused AsyncWebServerRequest *request) const override {`
	`60`	`+ bool canHandle(__asyncws_unused AsyncWebServerRequest *request) const override {`
`61`	`61`	`// Test backward compatibility with previous way of checking if a method is allowed in a composite using a bit operator`
`62`	`62`	`return allowed & request->method();`
`63`	`63`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#pragma once`
	`2`	`+`
	`3`	`+typedef void *SemaphoreHandle_t;`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#pragma once`
	`2`	`+`
	`3`	`+#define LWIP_IPV6 1`