[update] update vad & whisper

Author: Abandon-ht

…work/main_vad/mode_silero-vad-model.json …_framework/main_vad/mode_silero-vad.jsonprojects/llm_framework/main_vad/mode_silero-vad-model.json renamed to projects/llm_framework/main_vad/mode_silero-vad.json projects/llm_framework/main_vad/mode_silero-vad-model.json renamed to projects/llm_framework/main_vad/mode_silero-vad.json

@@ -32,6 +32,8 @@ static void __sigint(int iSigNo)
 static std::string base_model_path_;
 static std::string base_model_config_path_;
 
+typedef std::function<void(const bool &data)> task_callback_t;
+
 #define CONFIG_AUTO_SET(obj, key)             \
     if (config_body.contains(#key))           \
         mode_config_.key = config_body[#key]; \
@@ -50,16 +52,14 @@ class llm_task {
     bool enoutput_;
     bool enstream_;
     bool printed = false;
+    task_callback_t out_callback_;
     std::atomic_bool audio_flage_;
     int delay_audio_frame_ = 100;
     buffer_t *pcmdata;
     std::string wake_wav_file_;
 
-    std::function<void(const std::string &)> out_callback_;
-
     bool parse_config(const nlohmann::json &config_body)
     {
-        fprintf(stderr, "%s\n", mode_config_.ToString().c_str());
         try {
             model_           = config_body.at("model");
             response_format_ = config_body.at("response_format");
@@ -136,15 +136,14 @@ class llm_task {
         return 0;
     }
 
-    void set_output(std::function<void(const std::string &)> out_callback)
+    void set_output(task_callback_t out_callback)
     {
         out_callback_ = out_callback;
     }
 
     void sys_pcm_on_data(const std::string &raw)
     {
         static int count = 0;
-        int32_t k        = 0;
         if (count < delay_audio_frame_) {
             buffer_write_char(pcmdata, raw.c_str(), raw.length());
             count++;
@@ -167,6 +166,9 @@ class llm_task {
         if (vad_->IsSpeechDetected() && !printed) {
             printed = true;
             SLOGI("Detected speech!");
+            if (out_callback_) {
+                out_callback_(true);
+            }
         }
         if (!vad_->IsSpeechDetected()) {
             printed = false;
@@ -177,8 +179,11 @@ class llm_task {
             const auto &segment = vad_->Front();
             float duration      = segment.samples.size() / static_cast<float>(sample_rate);
             SLOGI("Duration: %.3f seconds", duration);
-            k += 1;
+            // k += 1;
             vad_->Pop();
+            if (out_callback_) {
+                out_callback_(false);
+            }
         }
     }
 
@@ -203,18 +208,31 @@ class llm_task {
 };
 #undef CONFIG_AUTO_SET
 
-class llm_kws : public StackFlow {
+class llm_vad : public StackFlow {
 private:
     int task_count_;
     std::string audio_url_;
     std::unordered_map<int, std::shared_ptr<llm_task>> llm_task_;
 
 public:
-    llm_kws() : StackFlow("vad")
+    llm_vad() : StackFlow("vad")
     {
         task_count_ = 1;
     }
 
+    void task_output(const std::weak_ptr<llm_task> llm_task_obj_weak,
+                     const std::weak_ptr<llm_channel_obj> llm_channel_weak, const bool &data)
+    {
+        auto llm_task_obj = llm_task_obj_weak.lock();
+        auto llm_channel  = llm_channel_weak.lock();
+        if (!(llm_task_obj && llm_channel)) {
+            return;
+        }
+        std::string tmp_msg1;
+        const bool *next_data = &data;
+        llm_channel->send(llm_task_obj->response_format_, (*next_data), LLM_NO_ERROR);
+    }
+
     void task_pause(const std::weak_ptr<llm_task> llm_task_obj_weak,
                     const std::weak_ptr<llm_channel_obj> llm_channel_weak)
     {
@@ -350,9 +368,8 @@ class llm_kws : public StackFlow {
         if (ret == 0) {
             llm_channel->set_output(llm_task_obj->enoutput_);
             llm_channel->set_stream(llm_task_obj->enstream_);
-            llm_task_obj->set_output([llm_task_obj, llm_channel](const std::string &data) {
-                llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR);
-            });
+            llm_task_obj->set_output(std::bind(&llm_vad::task_output, this, std::weak_ptr<llm_task>(llm_task_obj),
+                                               std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1));
 
             for (const auto input : llm_task_obj->inputs_) {
                 if (input.find("sys") != std::string::npos) {
@@ -364,7 +381,7 @@ class llm_kws : public StackFlow {
                     llm_task_obj->audio_flage_ = true;
                 } else if (input.find("vad") != std::string::npos) {
                     llm_channel->subscriber_work_id(
-                        "", std::bind(&llm_kws::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
+                        "", std::bind(&llm_vad::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
                                       std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
                                       std::placeholders::_2));
                 }
@@ -430,7 +447,7 @@ class llm_kws : public StackFlow {
         return 0;
     }
 
-    ~llm_kws()
+    ~llm_vad()
     {
         while (1) {
             auto iteam = llm_task_.begin();
@@ -452,7 +469,7 @@ int main(int argc, char *argv[])
     signal(SIGTERM, __sigint);
     signal(SIGINT, __sigint);
     mkdir("/tmp/llm", 0777);
-    llm_kws llm;
+    llm_vad llm;
     while (!main_exit_flage) {
         sleep(1);
     }

projects/llm_framework/main_vad/src/main.cpp

@@ -14,7 +14,7 @@
     ],
     "mode_param": {
         "model_type": "tiny",
-        "language": "zh",
+        "language": "ja",
         "encoder": "tiny-encoder.axmodel",
         "decoder_main": "tiny-decoder-main.axmodel",
         "decoder_loop": "tiny-decoder-loop.axmodel",

projects/llm_framework/main_whisper/mode_whisper-tiny.json

@@ -88,15 +88,15 @@ class llm_task {
     bool enoutput_;
     bool enstream_;
     bool ensleep_;
-    bool endpoint_;
     std::atomic_bool superior_flage_;
     std::atomic_bool audio_flage_;
     std::atomic_bool awake_flage_;
+    std::atomic_bool vad_endpoint_;
     std::string superior_id_;
     static int ax_init_flage_;
     task_callback_t out_callback_;
     int awake_delay_       = 50;
-    int delay_audio_frame_ = 100;
+    int delay_audio_frame_ = 1000;
     buffer_t *pcmdata;
 
     std::function<void(void)> pause;
@@ -301,23 +301,34 @@ class llm_task {
 
     void sys_pcm_on_data(const std::string &raw)
     {
+        static int count = 0;
+        if (count < delay_audio_frame_) {
+            buffer_write_char(pcmdata, raw.c_str(), raw.length());
+            count++;
+            return;
+        }
+        buffer_write_char(pcmdata, raw.c_str(), raw.length());
+        buffer_position_set(pcmdata, 0);
+        count = 0;
+        std::vector<float> floatSamples;
+        {
+            int16_t audio_val;
+            while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
+                float normalizedSample = (float)audio_val / INT16_MAX;
+                floatSamples.push_back(normalizedSample);
+            }
+        }
+        buffer_position_set(pcmdata, 0);
+
         if (WHISPER_N_TEXT_STATE_MAP.find(mode_config_.model_type) == WHISPER_N_TEXT_STATE_MAP.end()) {
             fprintf(stderr, "Can NOT find n_text_state for model_type: %s\n", mode_config_.model_type.c_str());
             return;
         }
 
         int WHISPER_N_TEXT_STATE = WHISPER_N_TEXT_STATE_MAP[mode_config_.model_type];
 
-        AudioFile<float> audio_file;
-        if (!audio_file.load("demo.wav")) {
-            printf("load wav failed!\n");
-            return;
-        }
-        awake_flage_  = false;
-        auto &samples = audio_file.samples[0];
-
         auto mel = librosa::Feature::melspectrogram(
-            samples, mode_config_.whisper_sample_rate, mode_config_.whisper_n_fft, mode_config_.whisper_hop_length,
+            floatSamples, mode_config_.whisper_sample_rate, mode_config_.whisper_n_fft, mode_config_.whisper_hop_length,
             "hann", true, "reflect", 2.0f, mode_config_.whisper_n_mels, 0.0f, mode_config_.whisper_sample_rate / 2.0f);
         int n_mel = mel.size();
         int n_len = mel[0].size();
@@ -457,57 +468,18 @@ class llm_task {
             s += str;
         }
 
-        if (mode_config_.language == "en")
+        if (mode_config_.language == "en" || mode_config_.language == "ja") {
             printf("Result: %s\n", s.c_str());
-        else {
+            if (out_callback_) out_callback_(s, true);
+        } else {
             const opencc::SimpleConverter converter(mode_config_.t2s.c_str());
             std::string simple_str = converter.Convert(s);
             printf("Result: %s\n", simple_str.c_str());
+            if ((!simple_str.empty()) && out_callback_) {
+                out_callback_(simple_str, true);
+            }
         }
-        /////////////////////////////////////////////////////////////////////
-        // static int count = 0;
-        // if (count < delay_audio_frame_) {
-        //     buffer_write_char(pcmdata, raw.c_str(), raw.length());
-        //     count++;
-        //     return;
-        // }
-        // buffer_write_char(pcmdata, raw.c_str(), raw.length());
-        // buffer_position_set(pcmdata, 0);
-        // count = 0;
-        // std::vector<float> floatSamples;
-        // {
-        //     int16_t audio_val;
-        //     while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
-        //         float normalizedSample = (float)audio_val / INT16_MAX;
-        //         floatSamples.push_back(normalizedSample);
-        //     }
-        // }
-        // buffer_position_set(pcmdata, 0);
-        // if (awake_flage_ && recognizer_stream_) {
-        //     recognizer_stream_.reset();
-        //     awake_flage_ = false;
-        // }
-        // if (!recognizer_stream_) {
-        //     recognizer_stream_ = recognizer_->CreateStream();
-        // }
-        // recognizer_stream_->AcceptWaveform(mode_config_.feat_config.sampling_rate, floatSamples.data(),
-        //                                    floatSamples.size());
-        // while (recognizer_->IsReady(recognizer_stream_.get())) {
-        //     recognizer_->DecodeStream(recognizer_stream_.get());
-        // }
-        // std::string text = recognizer_->GetResult(recognizer_stream_.get()).text;
-        // std::string lower_text;
-        // lower_text.resize(text.size());
-        // std::transform(text.begin(), text.end(), lower_text.begin(), [](const char c) { return std::tolower(c); });
-        // if ((!lower_text.empty()) && out_callback_) out_callback_(lower_text, false);
-        // bool is_endpoint = recognizer_->IsEndpoint(recognizer_stream_.get());
-        // if (is_endpoint) {
-        //     std::cout << "asr have a is_endpoint \n";
-        //     recognizer_stream_->Finalize();
-        //     if ((!lower_text.empty()) && out_callback_) {
-        //         out_callback_(lower_text, true);
-        //     }
-        //     recognizer_stream_.reset();
+
         if (ensleep_) {
             if (pause) pause();
         }
@@ -588,24 +560,24 @@ class llm_whisper : public StackFlow {
         if (!(llm_task_obj && llm_channel)) {
             return;
         }
-        std::string base64_data;
-        int len = encode_base64(data, base64_data);
+        std::string tmp_msg1;
+        const std::string *next_data = &data;
+        if (finish) {
+            tmp_msg1  = data + ".";
+            next_data = &tmp_msg1;
+        }
         if (llm_channel->enstream_) {
             static int count = 0;
             nlohmann::json data_body;
-            data_body["index"] = count++;
-            if (!finish)
-                data_body["delta"] = base64_data;
-            else
-                data_body["delta"] = std::string("");
+            data_body["index"]  = count++;
+            data_body["delta"]  = (*next_data);
             data_body["finish"] = finish;
             if (finish) count = 0;
+            SLOGI("send stream:%s", next_data->c_str());
             llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR);
         } else if (finish) {
-            llm_channel->send(llm_task_obj->response_format_, base64_data, LLM_NO_ERROR);
-        }
-        if (llm_task_obj->response_format_.find("sys") != std::string::npos) {
-            unit_call("audio", "queue_play", data);
+            SLOGI("send utf-8:%s", next_data->c_str());
+            llm_channel->send(llm_task_obj->response_format_, (*next_data), LLM_NO_ERROR);
         }
     }
 
@@ -744,6 +716,20 @@ class llm_whisper : public StackFlow {
         task_work(llm_task_obj, llm_channel);
     }
 
+    void vad_endpoint(const std::weak_ptr<llm_task> llm_task_obj_weak,
+                      const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &object,
+                      const std::string &data)
+    {
+        auto llm_task_obj = llm_task_obj_weak.lock();
+        auto llm_channel  = llm_channel_weak.lock();
+        if (!(llm_task_obj && llm_channel)) {
+            return;
+        }
+        if (data == "true" || data == "false") {
+            llm_task_obj->vad_endpoint_ = (data == "true");
+        }
+    }
+
     void work(const std::string &work_id, const std::string &object, const std::string &data) override
     {
         SLOGI("llm_asr::work:%s", data.c_str());
@@ -814,7 +800,7 @@ class llm_whisper : public StackFlow {
                     audio_url_                            = unit_call("audio", "cap", input);
                     std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
                     llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
-                        // _llm_task_obj.lock()->sys_pcm_on_data(raw);
+                        _llm_task_obj.lock()->sys_pcm_on_data(raw);
                     });
                     llm_task_obj->audio_flage_ = true;
                 } else if (input.find("asr") != std::string::npos) {
@@ -830,10 +816,10 @@ class llm_whisper : public StackFlow {
                                          std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
                                          std::placeholders::_2));
                 } else if (input.find("vad") != std::string::npos) {
-                    llm_task_obj->endpoint_ = true;
-                    task_pause(work_id, "");
+                    llm_task_obj->vad_endpoint_ = true;
+                    // task_pause(work_id, "");
                     llm_channel->subscriber_work_id(
-                        input, std::bind(&llm_whisper::kws_awake, this, std::weak_ptr<llm_task>(llm_task_obj),
+                        input, std::bind(&llm_whisper::vad_endpoint, this, std::weak_ptr<llm_task>(llm_task_obj),
                                          std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
                                          std::placeholders::_2));
                 }
@@ -880,6 +866,12 @@ class llm_whisper : public StackFlow {
                 std::bind(&llm_whisper::kws_awake, this, std::weak_ptr<llm_task>(llm_task_obj),
                                              std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1, std::placeholders::_2));
             llm_task_obj->inputs_.push_back(data);
+        } else if (data.find("vad") != std::string::npos) {
+            llm_task_obj->vad_endpoint_ = true;
+            ret                         = llm_channel->subscriber_work_id(
+                data,
+                std::bind(&llm_whisper::vad_endpoint, this, std::weak_ptr<llm_task>(llm_task_obj),
+                                                  std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1, std::placeholders::_2));
         }
         if (ret) {
             error_body["code"]    = -20;