audio fully works, GPU HRTF works as well as CPU HRTF.

Author: gpx1000

attachments/simple_engine/audio_system.cpp

@@ -4,6 +4,11 @@
 #include <vector>
 #include <memory>
 #include <unordered_map>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include <queue>
 #ifdef __INTELLISENSE__
 #include <vulkan/vulkan_raii.hpp>
 #else
@@ -267,7 +272,7 @@ class AudioSystem {
      * @param sampleCount The number of samples to generate.
      * @param playbackPosition The current playback position for timing.
      */
-    void GenerateSineWavePing(float* buffer, uint32_t sampleCount, uint32_t playbackPosition);
+    static void GenerateSineWavePing(float* buffer, uint32_t sampleCount, uint32_t playbackPosition);
 
 private:
     // Loaded audio data
@@ -300,6 +305,36 @@ class AudioSystem {
     // Audio output device for sending processed audio to speakers
     std::unique_ptr<AudioOutputDevice> outputDevice = nullptr;
 
+    // Threading infrastructure for background audio processing
+    std::thread audioThread;
+    std::mutex audioMutex;
+    std::condition_variable audioCondition;
+    std::atomic<bool> audioThreadRunning{false};
+    std::atomic<bool> audioThreadShouldStop{false};
+
+    // Audio processing task queue
+    struct AudioTask {
+        std::vector<float> inputBuffer;
+        std::vector<float> outputBuffer;
+        float sourcePosition[3];
+        uint32_t sampleCount;
+        uint32_t actualSamplesProcessed;
+        AudioOutputDevice* outputDevice;
+        float masterVolume;
+    };
+    // Set up HRTF parameters
+    struct HRTFParams {
+        float sourcePosition[3];
+        float listenerPosition[3];
+        float listenerOrientation[6]; // Forward (3) and up (3) vectors
+        uint32_t sampleCount;
+        uint32_t hrtfSize;
+        uint32_t numHrtfPositions;
+        float padding; // For alignment
+    } params;
+    std::queue<std::shared_ptr<AudioTask>> audioTaskQueue;
+    std::mutex taskQueueMutex;
+
     // Vulkan resources for HRTF processing
     vk::raii::Buffer inputBuffer = nullptr;
     vk::raii::DeviceMemory inputBufferMemory = nullptr;
@@ -310,6 +345,10 @@ class AudioSystem {
     vk::raii::Buffer paramsBuffer = nullptr;
     vk::raii::DeviceMemory paramsBufferMemory = nullptr;
 
+    // Persistent memory mapping for UBO to avoid repeated map/unmap operations
+    void* persistentParamsMemory = nullptr;
+    uint32_t currentSampleCount = 0; // Track current buffer size to avoid unnecessary recreation
+
     /**
      * @brief Create buffers for HRTF processing.
      * @param sampleCount The number of samples to process.
@@ -321,4 +360,36 @@ class AudioSystem {
      * @brief Clean up HRTF buffers.
      */
     void cleanupHRTFBuffers();
+
+
+    /**
+     * @brief Start the background audio processing thread.
+     */
+    void startAudioThread();
+
+    /**
+     * @brief Stop the background audio processing thread.
+     */
+    void stopAudioThread();
+
+    /**
+     * @brief Main loop for the background audio processing thread.
+     */
+    void audioThreadLoop();
+
+    /**
+     * @brief Process an audio task in the background thread.
+     * @param task The audio task to process.
+     */
+    void processAudioTask(const std::shared_ptr<AudioTask>& task);
+
+    /**
+     * @brief Submit an audio processing task to the background thread.
+     * @param inputBuffer The input audio buffer.
+     * @param sampleCount The number of samples to process.
+     * @param sourcePosition The position of the sound source.
+     * @param actualSamplesProcessed The number of samples actually processed.
+     * @return True if the task was submitted successfully, false otherwise.
+     */
+    bool submitAudioTask(const float* inputBuffer, uint32_t sampleCount, const float* sourcePosition, uint32_t actualSamplesProcessed);
 };

attachments/simple_engine/audio_system.h

@@ -11,6 +11,7 @@ import vulkan_hpp;
 #include <string>
 #include <optional>
 #include <unordered_map>
+#include <mutex>
 
 #include "platform.h"
 #include "entity.h"
@@ -119,10 +120,11 @@ class Renderer {
      * @param outputBuffer The output buffer.
      * @param hrtfBuffer The HRTF data buffer.
      * @param paramsBuffer The parameters buffer.
+     * @return A fence that can be used to synchronize with the compute operation.
      */
-    void DispatchCompute(uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ,
-                        vk::Buffer inputBuffer, vk::Buffer outputBuffer,
-                        vk::Buffer hrtfBuffer, vk::Buffer paramsBuffer);
+    vk::raii::Fence DispatchCompute(uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ,
+                                   vk::Buffer inputBuffer, vk::Buffer outputBuffer,
+                                   vk::Buffer hrtfBuffer, vk::Buffer paramsBuffer);
 
     /**
      * @brief Check if the renderer is initialized.
@@ -146,7 +148,10 @@ class Renderer {
      * @brief Get the compute queue.
      * @return The compute queue.
      */
-    vk::Queue GetComputeQueue() const { return *computeQueue; }
+    vk::Queue GetComputeQueue() const {
+        std::lock_guard<std::mutex> lock(queueMutex);
+        return *computeQueue;
+    }
 
     /**
      * @brief Find a suitable memory type.
@@ -158,6 +163,32 @@ class Renderer {
         return findMemoryType(typeFilter, properties);
     }
 
+    /**
+     * @brief Get the compute queue family index.
+     * @return The compute queue family index.
+     */
+    uint32_t GetComputeQueueFamilyIndex() const {
+        return queueFamilyIndices.computeFamily.value();
+    }
+
+    /**
+     * @brief Submit a command buffer to the compute queue with proper dispatch loader preservation.
+     * @param commandBuffer The command buffer to submit.
+     * @param fence The fence to signal when the operation completes.
+     */
+    void SubmitToComputeQueue(vk::CommandBuffer commandBuffer, vk::Fence fence) {
+        vk::SubmitInfo submitInfo{
+            .commandBufferCount = 1,
+            .pCommandBuffers = &commandBuffer
+        };
+
+        // Use mutex to ensure thread-safe access to compute queue
+        {
+            std::lock_guard<std::mutex> lock(queueMutex);
+            computeQueue.submit(submitInfo, fence);
+        }
+    }
+
     /**
      * @brief Create a shader module from SPIR-V code.
      * @param code The SPIR-V code.
@@ -278,6 +309,10 @@ class Renderer {
     vk::raii::DescriptorSetLayout computeDescriptorSetLayout = nullptr;
     vk::raii::DescriptorPool computeDescriptorPool = nullptr;
     std::vector<vk::raii::DescriptorSet> computeDescriptorSets;
+    vk::raii::CommandPool computeCommandPool = nullptr;
+
+    // Thread safety for queue access - unified mutex since queues may share the same underlying VkQueue
+    mutable std::mutex queueMutex;
 
     // Command pool and buffers
     vk::raii::CommandPool commandPool = nullptr;
@@ -385,6 +420,7 @@ class Renderer {
     bool createComputePipeline();
     void pushMaterialProperties(vk::CommandBuffer commandBuffer, const MaterialProperties& material);
     bool createCommandPool();
+    bool createComputeCommandPool();
     bool createDepthResources();
     bool createTextureImage(const std::string& texturePath, TextureResources& resources);
     bool createTextureImageView(TextureResources& resources);

attachments/simple_engine/renderer.h

@@ -9,7 +9,7 @@
 bool Renderer::createComputePipeline() {
     try {
         // Read compute shader code
-        auto computeShaderCode = readFile("shaders/compute.spv");
+        auto computeShaderCode = readFile("shaders/hrtf.spv");
 
         // Create shader module
         vk::raii::ShaderModule computeShaderModule = createShaderModule(computeShaderCode);
@@ -99,18 +99,38 @@ bool Renderer::createComputePipeline() {
 
         computeDescriptorPool = vk::raii::DescriptorPool(device, poolInfo);
 
-        return true;
+        return createComputeCommandPool();
     } catch (const std::exception& e) {
         std::cerr << "Failed to create compute pipeline: " << e.what() << std::endl;
         return false;
     }
 }
 
+// Create compute command pool
+bool Renderer::createComputeCommandPool() {
+    try {
+        vk::CommandPoolCreateInfo poolInfo{
+            .flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer,
+            .queueFamilyIndex = queueFamilyIndices.computeFamily.value()
+        };
+
+        computeCommandPool = vk::raii::CommandPool(device, poolInfo);
+        return true;
+    } catch (const std::exception& e) {
+        std::cerr << "Failed to create compute command pool: " << e.what() << std::endl;
+        return false;
+    }
+}
+
 // Dispatch compute shader
-void Renderer::DispatchCompute(uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ,
-                              vk::Buffer inputBuffer, vk::Buffer outputBuffer,
-                              vk::Buffer hrtfBuffer, vk::Buffer paramsBuffer) {
+vk::raii::Fence Renderer::DispatchCompute(uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ,
+                                         vk::Buffer inputBuffer, vk::Buffer outputBuffer,
+                                         vk::Buffer hrtfBuffer, vk::Buffer paramsBuffer) {
     try {
+        // Create fence for synchronization
+        vk::FenceCreateInfo fenceInfo{};
+        vk::raii::Fence computeFence(device, fenceInfo);
+
         // Create descriptor sets
         vk::DescriptorSetAllocateInfo allocInfo{
             .descriptorPool = *computeDescriptorPool,
@@ -182,47 +202,62 @@ void Renderer::DispatchCompute(uint32_t groupCountX, uint32_t groupCountY, uint3
 
         device.updateDescriptorSets(descriptorWrites, {});
 
-        // Create command buffer
+        // Create command buffer using dedicated compute command pool
         vk::CommandBufferAllocateInfo cmdAllocInfo{
-            .commandPool = *commandPool,
+            .commandPool = *computeCommandPool,
             .level = vk::CommandBufferLevel::ePrimary,
             .commandBufferCount = 1
         };
 
         auto commandBuffers = device.allocateCommandBuffers(cmdAllocInfo);
-        vk::CommandBuffer cmdBuffer = commandBuffers[0];
-        vk::raii::CommandBuffer commandBuffer(device, cmdBuffer, *commandPool);
+        // Use RAII wrapper temporarily for recording to preserve dispatch loader
+        vk::raii::CommandBuffer commandBufferRaii = std::move(commandBuffers[0]);
 
         // Begin command buffer
         vk::CommandBufferBeginInfo beginInfo{
             .flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit
         };
 
-        commandBuffer.begin(beginInfo);
+        commandBufferRaii.begin(beginInfo);
 
         // Bind compute pipeline
-        commandBuffer.bindPipeline(vk::PipelineBindPoint::eCompute, *computePipeline);
+        commandBufferRaii.bindPipeline(vk::PipelineBindPoint::eCompute, *computePipeline);
 
-        // Bind descriptor sets
-        commandBuffer.bindDescriptorSets(vk::PipelineBindPoint::eCompute, *computePipelineLayout, 0, reinterpret_cast<std::vector<vk::DescriptorSet> &>(computeDescriptorSets), {});
+        // Bind descriptor sets - properly convert RAII descriptor set to regular descriptor set
+        std::vector<vk::DescriptorSet> descriptorSetsToBindRaw;
+        descriptorSetsToBindRaw.reserve(1);
+        descriptorSetsToBindRaw.push_back(*computeDescriptorSets[0]);
+        commandBufferRaii.bindDescriptorSets(vk::PipelineBindPoint::eCompute, *computePipelineLayout, 0, descriptorSetsToBindRaw, {});
 
         // Dispatch compute shader
-        commandBuffer.dispatch(groupCountX, groupCountY, groupCountZ);
+        commandBufferRaii.dispatch(groupCountX, groupCountY, groupCountZ);
 
         // End command buffer
-        commandBuffer.end();
+        commandBufferRaii.end();
+
+        // Extract raw command buffer for submission and release RAII ownership
+        // This prevents premature destruction while preserving the recorded commands
+        vk::CommandBuffer rawCommandBuffer = *commandBufferRaii;
+        commandBufferRaii.release(); // Release RAII ownership to prevent destruction
 
-        // Submit command buffer
+        // Submit command buffer with fence for synchronization
         vk::SubmitInfo submitInfo{
             .commandBufferCount = 1,
-            .pCommandBuffers = &*commandBuffer
+            .pCommandBuffers = &rawCommandBuffer
         };
 
-        computeQueue.submit(submitInfo, nullptr);
+        // Use mutex to ensure thread-safe access to compute queue
+        {
+            std::lock_guard<std::mutex> lock(queueMutex);
+            computeQueue.submit(submitInfo, *computeFence);
+        }
 
-        // Wait for compute to complete
-        computeQueue.waitIdle();
+        // Return fence for non-blocking synchronization
+        return computeFence;
     } catch (const std::exception& e) {
         std::cerr << "Failed to dispatch compute shader: " << e.what() << std::endl;
+        // Return a null fence on error
+        vk::FenceCreateInfo fenceInfo{};
+        return vk::raii::Fence(device, fenceInfo);
     }
 }

attachments/simple_engine/renderer_compute.cpp

@@ -21,10 +21,21 @@ static VKAPI_ATTR VkBool32 VKAPI_CALL debugCallbackVkRaii(
     const vk::DebugUtilsMessengerCallbackDataEXT* pCallbackData,
     void* pUserData) {
 
-    if (messageSeverity >= vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning) {
-        // Print message to console
-        std::cerr << "Validation layer: " << pCallbackData->pMessage << std::endl;
-    }
+    // // Check if this is a shader debug printf message
+    // if (messageType & vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation) {
+    //     std::string message(pCallbackData->pMessage);
+    //     if (message.find("DEBUG-PRINTF") != std::string::npos) {
+    //         // This is a shader debug printf message - always show it
+    //         std::cout << "FINDME =====   SHADER DEBUG: " << pCallbackData->pMessage << std::endl;
+    //         return VK_FALSE;
+    //     }
+    // }
+    printf("Received %s\n", pCallbackData->pMessage);
+
+    // if (messageSeverity >= vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning) {
+    //     // Print message to console
+    //     std::cerr << "Validation layer: " << pCallbackData->pMessage << std::endl;
+    // }
 
     return VK_FALSE;
 }
@@ -222,6 +233,9 @@ bool Renderer::createInstance(const std::string& appName, bool enableValidationL
         };
 
         // Enable validation layers if requested
+        vk::ValidationFeaturesEXT validationFeatures{};
+        std::vector<vk::ValidationFeatureEnableEXT> enabledValidationFeatures;
+
         if (enableValidationLayers) {
             if (!checkValidationLayerSupport()) {
                 std::cerr << "Validation layers requested, but not available" << std::endl;
@@ -230,6 +244,14 @@ bool Renderer::createInstance(const std::string& appName, bool enableValidationL
 
             createInfo.enabledLayerCount = static_cast<uint32_t>(validationLayers.size());
             createInfo.ppEnabledLayerNames = validationLayers.data();
+
+            // Enable debug printf functionality for shader debugging
+            enabledValidationFeatures.push_back(vk::ValidationFeatureEnableEXT::eDebugPrintf);
+
+            validationFeatures.enabledValidationFeatureCount = static_cast<uint32_t>(enabledValidationFeatures.size());
+            validationFeatures.pEnabledValidationFeatures = enabledValidationFeatures.data();
+
+            createInfo.pNext = &validationFeatures;
         }
 
         // Create instance
@@ -251,6 +273,7 @@ bool Renderer::setupDebugMessenger(bool enableValidationLayers) {
         // Create debug messenger info
         vk::DebugUtilsMessengerCreateInfoEXT createInfo{
             .messageSeverity = vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose |
+                              vk::DebugUtilsMessageSeverityFlagBitsEXT::eInfo |
                               vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning |
                               vk::DebugUtilsMessageSeverityFlagBitsEXT::eError,
             .messageType = vk::DebugUtilsMessageTypeFlagBitsEXT::eGeneral |