Skip to content

Commit a151aff

Browse files
yuyun2000yuyun2000
committed
Translate logs in Lexicon.hpp to English and add debug switch
- Convert all Chinese log messages in Lexicon.hpp to English for better international compatibility\n- Add a debug flag to control whether to display g2p process logs\n- Improve code readability and debugging experience
1 parent e3c70bc commit a151aff

1 file changed

Lines changed: 52 additions & 52 deletions

File tree

  • projects/llm_framework/main_melotts/src/runner

‎projects/llm_framework/main_melotts/src/runner/Lexicon.hpp‎

Lines changed: 52 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#pragma once
2-
32
#include <string>
43
#include <vector>
54
#include <fstream>
@@ -9,7 +8,15 @@
98
#include <cassert>
109
#include <iostream>
1110
#include "../../../../../SDK/components/utilities/include/sample_log.h"
12-
11+
// Debug logging switch - set to true to enable debug logs
12+
static bool DEBUG_LOGGING = false;
13+
// Macro for debug logging
14+
#define DEBUG_LOG(fmt, ...) \
15+
do { \
16+
if (DEBUG_LOGGING) { \
17+
SLOGI(fmt, ##__VA_ARGS__); \
18+
} \
19+
} while (0)
1320
std::vector<std::string> split(const std::string& s, char delim)
1421
{
1522
std::vector<std::string> result;
@@ -30,9 +37,16 @@ class Lexicon {
3037
std::unordered_map<int, std::string> reverse_tokens;
3138

3239
public:
40+
// Setter for debug logging
41+
static void setDebugLogging(bool enable)
42+
{
43+
DEBUG_LOGGING = enable;
44+
}
3345
Lexicon(const std::string& lexicon_filename, const std::string& tokens_filename) : max_phrase_length(0)
3446
{
35-
SLOGI("词典加载: %zu 发音表加载: %zu", tokens_filename, lexicon_filename);
47+
DEBUG_LOG("Dictionary loading: %s Pronunciation table loading: %s", tokens_filename.c_str(),
48+
lexicon_filename.c_str());
49+
3650
std::unordered_map<std::string, int> tokens;
3751
std::ifstream ifs(tokens_filename);
3852
assert(ifs.is_open());
@@ -83,8 +97,10 @@ class Lexicon {
8397
lexicon["。"] = lexicon["."];
8498
lexicon["！"] = lexicon["!"];
8599
lexicon["？"] = lexicon["?"];
86-
SLOGI("词典加载完成，包含 %zu 个条目，最长词组长度: %zu", lexicon.size(), max_phrase_length);
100+
DEBUG_LOG("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(),
101+
max_phrase_length);
87102
}
103+
88104
std::vector<std::string> splitEachChar(const std::string& text)
89105
{
90106
std::vector<std::string> words;
@@ -95,93 +111,77 @@ class Lexicon {
95111
if ((text[i] & 0x80) == 0x00) {
96112
// ASCII
97113
} else if ((text[i] & 0xE0) == 0xC0) {
98-
next = 2; // 2字节UTF-8
114+
next = 2; // 2-byte UTF-8
99115
} else if ((text[i] & 0xF0) == 0xE0) {
100-
next = 3; // 3字节UTF-8
116+
next = 3; // 3-byte UTF-8
101117
} else if ((text[i] & 0xF8) == 0xF0) {
102-
next = 4; // 4字节UTF-8
118+
next = 4; // 4-byte UTF-8
103119
}
104120
words.push_back(text.substr(i, next));
105121
i += next;
106122
}
107123
return words;
108124
}
125+
109126
bool is_english(const std::string& s)
110127
{
111128
return s.size() == 1 && ((s[0] >= 'A' && s[0] <= 'Z') || (s[0] >= 'a' && s[0] <= 'z'));
112129
}
113-
114130
bool is_english_token_char(const std::string& s)
115131
{
116132
if (s.size() != 1) return false;
117133
char c = s[0];
118134
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_';
119135
}
120-
121136
void process_unknown_english(const std::string& word, std::vector<int>& phones, std::vector<int>& tones)
122137
{
123-
SLOGI("Processing unknown term: %s", word.c_str());
124-
138+
DEBUG_LOG("Processing unknown term: %s", word.c_str());
125139
std::string orig_word = word;
126140
std::vector<std::string> parts;
127141
std::vector<std::string> phonetic_parts;
128-
129142
size_t start = 0;
130143
while (start < word.size()) {
131144
bool matched = false;
132-
133145
for (size_t len = std::min(word.size() - start, (size_t)10); len > 0 && !matched; --len) {
134146
std::string sub_word = word.substr(start, len);
135147
std::string lower_sub_word = sub_word;
136148
std::transform(lower_sub_word.begin(), lower_sub_word.end(), lower_sub_word.begin(),
137149
[](unsigned char c) { return std::tolower(c); });
138-
139150
if (lexicon.find(lower_sub_word) != lexicon.end()) {
140151
// Substring found in lexicon
141152
auto& [sub_phones, sub_tones] = lexicon[lower_sub_word];
142153
phones.insert(phones.end(), sub_phones.begin(), sub_phones.end());
143154
tones.insert(tones.end(), sub_tones.begin(), sub_tones.end());
144-
145155
parts.push_back(sub_word);
146156
phonetic_parts.push_back(phonesToString(sub_phones));
147-
148-
SLOGI(" Matched: '%s' -> %s", sub_word.c_str(), phonesToString(sub_phones).c_str());
149-
157+
DEBUG_LOG(" Matched: '%s' -> %s", sub_word.c_str(), phonesToString(sub_phones).c_str());
150158
start += len;
151159
matched = true;
152160
break;
153161
}
154162
}
155-
156163
if (!matched) {
157164
std::string single_char = word.substr(start, 1);
158165
std::string lower_char = single_char;
159166
std::transform(lower_char.begin(), lower_char.end(), lower_char.begin(),
160167
[](unsigned char c) { return std::tolower(c); });
161-
162168
if (lexicon.find(lower_char) != lexicon.end()) {
163169
auto& [char_phones, char_tones] = lexicon[lower_char];
164170
phones.insert(phones.end(), char_phones.begin(), char_phones.end());
165171
tones.insert(tones.end(), char_tones.begin(), char_tones.end());
166-
167172
parts.push_back(single_char);
168173
phonetic_parts.push_back(phonesToString(char_phones));
169-
170-
SLOGI(" Single char: '%s' -> %s", single_char.c_str(), phonesToString(char_phones).c_str());
174+
DEBUG_LOG(" Single char: '%s' -> %s", single_char.c_str(), phonesToString(char_phones).c_str());
171175
} else {
172176
phones.insert(phones.end(), unknown_token.first.begin(), unknown_token.first.end());
173177
tones.insert(tones.end(), unknown_token.second.begin(), unknown_token.second.end());
174-
175178
parts.push_back(single_char);
176179
phonetic_parts.push_back("_unknown_");
177-
178-
SLOGI(" Unknown: '%s'", single_char.c_str());
180+
DEBUG_LOG(" Unknown: '%s'", single_char.c_str());
179181
}
180-
181182
start++;
182183
}
183184
}
184-
185185
std::string parts_str, phonetic_str;
186186
for (size_t i = 0; i < parts.size(); i++) {
187187
if (i > 0) {
@@ -191,20 +191,20 @@ class Lexicon {
191191
parts_str += parts[i];
192192
phonetic_str += phonetic_parts[i];
193193
}
194-
195-
SLOGI("%s\t|\tDecomposed: %s\t|\tPhonetics: %s", orig_word.c_str(), parts_str.c_str(), phonetic_str.c_str());
194+
DEBUG_LOG("%s\t|\tDecomposed: %s\t|\tPhonetics: %s", orig_word.c_str(), parts_str.c_str(),
195+
phonetic_str.c_str());
196196
}
197+
197198
void convert(const std::string& text, std::vector<int>& phones, std::vector<int>& tones)
198199
{
199-
SLOGI("\n开始处理文本: \"%s\"", text.c_str());
200-
SLOGI("=======匹配结果=======");
201-
SLOGI("单元\t|\t音素\t|\t声调");
202-
SLOGI("-----------------------------");
200+
DEBUG_LOG("\nStarting text processing: \"%s\"", text.c_str());
201+
DEBUG_LOG("=======Matching Results=======");
202+
DEBUG_LOG("Unit\t|\tPhonemes\t|\tTones");
203+
DEBUG_LOG("-----------------------------");
203204
phones.insert(phones.end(), unknown_token.first.begin(), unknown_token.first.end());
204205
tones.insert(tones.end(), unknown_token.second.begin(), unknown_token.second.end());
205-
206-
SLOGI("<BOS>\t|\t%s\t|\t%s", phonesToString(unknown_token.first).c_str(),
207-
tonesToString(unknown_token.second).c_str());
206+
DEBUG_LOG("<BOS>\t|\t%s\t|\t%s", phonesToString(unknown_token.first).c_str(),
207+
tonesToString(unknown_token.second).c_str());
208208
auto chars = splitEachChar(text);
209209
int i = 0;
210210
while (i < chars.size()) {
@@ -221,8 +221,8 @@ class Lexicon {
221221
auto& [eng_phones, eng_tones] = lexicon[eng_word];
222222
phones.insert(phones.end(), eng_phones.begin(), eng_phones.end());
223223
tones.insert(tones.end(), eng_tones.begin(), eng_tones.end());
224-
SLOGI("%s\t|\t%s\t|\t%s", orig_word.c_str(), phonesToString(eng_phones).c_str(),
225-
tonesToString(eng_tones).c_str());
224+
DEBUG_LOG("%s\t|\t%s\t|\t%s", orig_word.c_str(), phonesToString(eng_phones).c_str(),
225+
tonesToString(eng_tones).c_str());
226226
} else {
227227
process_unknown_english(orig_word, phones, tones);
228228
}
@@ -241,8 +241,8 @@ class Lexicon {
241241
auto& [phrase_phones, phrase_tones] = lexicon[phrase];
242242
phones.insert(phones.end(), phrase_phones.begin(), phrase_phones.end());
243243
tones.insert(tones.end(), phrase_tones.begin(), phrase_tones.end());
244-
SLOGI("%s\t|\t%s\t|\t%s", phrase.c_str(), phonesToString(phrase_phones).c_str(),
245-
tonesToString(phrase_tones).c_str());
244+
DEBUG_LOG("%s\t|\t%s\t|\t%s", phrase.c_str(), phonesToString(phrase_phones).c_str(),
245+
tonesToString(phrase_tones).c_str());
246246
i += len;
247247
matched = true;
248248
break;
@@ -264,25 +264,25 @@ class Lexicon {
264264
auto& [char_phones, char_tones] = lexicon[s];
265265
phones.insert(phones.end(), char_phones.begin(), char_phones.end());
266266
tones.insert(tones.end(), char_tones.begin(), char_tones.end());
267-
SLOGI("%s\t|\t%s\t|\t%s", orig_char.c_str(), phonesToString(char_phones).c_str(),
268-
tonesToString(char_tones).c_str());
267+
DEBUG_LOG("%s\t|\t%s\t|\t%s", orig_char.c_str(), phonesToString(char_phones).c_str(),
268+
tonesToString(char_tones).c_str());
269269
} else {
270270
phones.insert(phones.end(), unknown_token.first.begin(), unknown_token.first.end());
271271
tones.insert(tones.end(), unknown_token.second.begin(), unknown_token.second.end());
272-
SLOGI("%s\t|\t%s (未匹配)\t|\t%s", orig_char.c_str(), phonesToString(unknown_token.first).c_str(),
273-
tonesToString(unknown_token.second).c_str());
272+
DEBUG_LOG("%s\t|\t%s (Not matched)\t|\t%s", orig_char.c_str(),
273+
phonesToString(unknown_token.first).c_str(), tonesToString(unknown_token.second).c_str());
274274
}
275275
}
276276
}
277277
phones.insert(phones.end(), unknown_token.first.begin(), unknown_token.first.end());
278278
tones.insert(tones.end(), unknown_token.second.begin(), unknown_token.second.end());
279-
SLOGI("<EOS>\t|\t%s\t|\t%s", phonesToString(unknown_token.first).c_str(),
280-
tonesToString(unknown_token.second).c_str());
281-
SLOGI("\n处理结果汇总:");
282-
SLOGI("原文: %s", text.c_str());
283-
SLOGI("音素: %s", phonesToString(phones).c_str());
284-
SLOGI("声调: %s", tonesToString(tones).c_str());
285-
SLOGI("====================");
279+
DEBUG_LOG("<EOS>\t|\t%s\t|\t%s", phonesToString(unknown_token.first).c_str(),
280+
tonesToString(unknown_token.second).c_str());
281+
DEBUG_LOG("\nProcessing Summary:");
282+
DEBUG_LOG("Original text: %s", text.c_str());
283+
DEBUG_LOG("Phonemes: %s", phonesToString(phones).c_str());
284+
DEBUG_LOG("Tones: %s", tonesToString(tones).c_str());
285+
DEBUG_LOG("====================");
286286
}
287287

288288
private:

0 commit comments

Comments
 (0)