Skip to content

Commit 8d53896

Browse files
dianjixzdianjixz
committed
Merge branch 'dev' of github.com:m5stack/StackFlow into dev
2 parents 2b9bda4 + 869c827 commit 8d53896

5 files changed

Lines changed: 147 additions & 14 deletions

File tree

‎projects/llm_framework/main_kws/src/main.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ class llm_task {
178178
if (file_exists("/opt/m5stack/scripts/text2token.py"))
179179
awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/text2token.py ";
180180
else if (file_exists("/opt/m5stack/scripts/llm-kws_text2token.py"))
181-
awake_key_compile_cmd << "/bin/bash /opt/m5stack/scripts/llm-kws_text2token.py ";
181+
awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/llm-kws_text2token.py ";
182182
else {
183183
SLOGE("text2token.py or llm-kws_text2token.py not found!");
184184
}

‎projects/llm_framework/main_llm/mode_deepseek-r1-1.5B-ax630c.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@
2424
"b_use_topk":false,
2525
"b_bos":false,
2626
"b_eos":false,
27-
"axmodel_num":24,
27+
"axmodel_num":28,
2828
"tokens_embed_num":151936,
29-
"tokens_embed_size":896,
29+
"tokens_embed_size":1536,
3030
"b_use_mmap_load_embed":true,
3131
"b_dynamic_load_axmodel_layer":false,
32-
"ext_scripts":["tokenizer_qwen2.5-1.5B-ax630c.py"]
32+
"ext_scripts":["tokenizer_deepseek-r1-1.5B-ax630c.py"]
3333
}
3434
}

‎projects/llm_framework/main_llm/tokenizer_deepseek-r1-1.5B-ax630c.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
from transformers import AutoTokenizer, PreTrainedTokenizerFast
2+
from http.server import HTTPServer, BaseHTTPRequestHandler
3+
import json
4+
import argparse
5+
6+
class Tokenizer_Http():
7+
8+
def __init__(self, model_id):
9+
self.tokenizer = AutoTokenizer.from_pretrained(model_id)
10+
11+
def encode(self, prompt, content):
12+
messages = [
13+
{"role": "system", "content": content},
14+
{"role": "user", "content": prompt}
15+
]
16+
text = self.tokenizer.apply_chat_template(
17+
messages,
18+
tokenize=False,
19+
add_generation_prompt=True
20+
)
21+
print(text)
22+
token_ids = self.tokenizer.encode(text)
23+
return token_ids
24+
25+
def decode(self, token_ids):
26+
return self.tokenizer.decode(token_ids)
27+
28+
@property
29+
def bos_id(self):
30+
return self.tokenizer.bos_token_id
31+
32+
@property
33+
def eos_id(self):
34+
return self.tokenizer.eos_token_id
35+
36+
@property
37+
def bos_token(self):
38+
return self.tokenizer.bos_token
39+
40+
@property
41+
def eos_token(self):
42+
return self.tokenizer.eos_token
43+
44+
class Request(BaseHTTPRequestHandler):
45+
#通过类继承，新定义类
46+
timeout = 5
47+
server_version = 'Apache'
48+
49+
def do_GET(self):
50+
print(self.path)
51+
#在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
52+
self.send_response(200)
53+
self.send_header("type", "get") #设置响应头，可省略或设置多个
54+
self.end_headers()
55+
56+
if self.path == '/bos_id':
57+
bos_id = tokenizer.bos_id
58+
# print(bos_id)
59+
# to json
60+
if bos_id is None:
61+
msg = json.dumps({'bos_id': -1})
62+
else:
63+
msg = json.dumps({'bos_id': bos_id})
64+
elif self.path == '/eos_id':
65+
eos_id = tokenizer.eos_id
66+
if eos_id is None:
67+
msg = json.dumps({'eos_id': -1})
68+
else:
69+
msg = json.dumps({'eos_id': eos_id})
70+
else:
71+
msg = 'error'
72+
73+
print(msg)
74+
msg = str(msg).encode() #转为str再转为byte格式
75+
76+
self.wfile.write(msg) #将byte格式的信息返回给客户端
77+
78+
def do_POST(self):
79+
#在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
80+
data = self.rfile.read(int(
81+
self.headers['content-length'])) #获取从客户端传入的参数（byte格式）
82+
data = data.decode() #将byte格式转为str格式
83+
84+
self.send_response(200)
85+
self.send_header("type", "post") #设置响应头，可省略或设置多个
86+
self.end_headers()
87+
88+
if self.path == '/encode':
89+
req = json.loads(data)
90+
prompt = req['text']
91+
92+
token_ids = tokenizer.encode(prompt, args.content)
93+
if token_ids is None:
94+
msg = json.dumps({'token_ids': -1})
95+
else:
96+
msg = json.dumps({'token_ids': token_ids})
97+
98+
elif self.path == '/decode':
99+
req = json.loads(data)
100+
token_ids = req['token_ids']
101+
text = tokenizer.decode(token_ids)
102+
if text is None:
103+
msg = json.dumps({'text': ""})
104+
else:
105+
msg = json.dumps({'text': text})
106+
else:
107+
msg = 'error'
108+
print(msg)
109+
msg = str(msg).encode() #转为str再转为byte格式
110+
111+
self.wfile.write(msg) #将byte格式的信息返回给客户端
112+
113+
114+
if __name__ == "__main__":
115+
116+
args = argparse.ArgumentParser()
117+
args.add_argument('--host', type=str, default='localhost')
118+
args.add_argument('--port', type=int, default=8080)
119+
args.add_argument('--model_id', type=str, default='deepseek_tokenizer')
120+
args.add_argument('--content', type=str, default='You are a helpful assistant.')
121+
args = args.parse_args()
122+
123+
tokenizer = Tokenizer_Http(args.model_id)
124+
125+
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
126+
# print(tokenizer.encode("hello world", args.content))
127+
128+
host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1'
129+
print('http://%s:%s' % host)
130+
server = HTTPServer(host, Request) #根据地址端口号和新定义的类，创建服务器实例
131+
server.serve_forever() #开启服务

‎projects/llm_framework/main_vlm/mode_internvl2.5-1B-ax630c.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"filename_tokenizer_model":"http://localhost:8080",
1919
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
2020
"filename_post_axmodel":"qwen2_post.axmodel",
21-
"template_filename_axmodel":"qwen2_p128_l%d_together.axmodel",
21+
"template_filename_axmodel":"qwen2_p256_l%d_together.axmodel",
2222
"filename_vpm_resampler_axmodedl":"intervl_vision_part_224.axmodel",
2323
"b_use_topk":false,
2424
"b_bos":false,

‎projects/llm_framework/tools/llm_pack.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'):
286286
create_lib = True
287287
create_bin = True
288288
create_data = True
289+
create_llm_data = True
289290
if len(sys.argv) > 1:
290291
src_folder = sys.argv[1]
291292
cpu_count = os.cpu_count()
@@ -320,24 +321,25 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'):
320321
futures.append(executor.submit(create_data_deb,'llm-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23', data_version, src_folder, revision))
321322
futures.append(executor.submit(create_data_deb,'llm-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01', '0.3', src_folder, revision))
322323
futures.append(executor.submit(create_data_deb,'llm-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01', '0.3', src_folder, revision))
323-
# futures.append(executor.submit(create_data_deb,'llm-qwen2-0.5B-prefill-20e', data_version, src_folder, revision))
324-
# futures.append(executor.submit(create_data_deb,'llm-qwen2-1.5B-prefill-20e', data_version, src_folder, revision))
325-
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision))
326-
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-1.5B-ax630c', '0.3', src_folder, revision))
327324
futures.append(executor.submit(create_data_deb,'llm-single-speaker-english-fast', data_version, src_folder, revision))
328325
futures.append(executor.submit(create_data_deb,'llm-single-speaker-fast', data_version, src_folder, revision))
329-
futures.append(executor.submit(create_data_deb,'llm-melotts-zh-cn', '0.3', src_folder, revision))
326+
futures.append(executor.submit(create_data_deb,'llm-melotts-zh-cn', '0.4', src_folder, revision))
330327
futures.append(executor.submit(create_data_deb,'llm-yolo11n', data_version, src_folder, revision))
331328
futures.append(executor.submit(create_data_deb,'llm-yolo11n-pose', '0.3', src_folder, revision))
332329
futures.append(executor.submit(create_data_deb,'llm-yolo11n-hand-pose', '0.3', src_folder, revision))
333330
futures.append(executor.submit(create_data_deb,'llm-yolo11n-seg', data_version, src_folder, revision))
334-
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision))
335-
futures.append(executor.submit(create_data_deb,'llm-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision))
336-
futures.append(executor.submit(create_data_deb,'llm-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision))
337-
futures.append(executor.submit(create_data_deb,'llm-internvl2.5-1B-ax630c', '0.3', src_folder, revision))
338331
futures.append(executor.submit(create_data_deb,'llm-depth-anything-ax630c', '0.3', src_folder, revision))
339332
futures.append(executor.submit(create_data_deb,'llm-whisper-tiny', '0.3', src_folder, revision))
340333
futures.append(executor.submit(create_data_deb,'llm-silero-vad', '0.3', src_folder, revision))
334+
if (create_llm_data):
335+
# futures.append(executor.submit(create_data_deb,'llm-qwen2-0.5B-prefill-20e', data_version, src_folder, revision))
336+
# futures.append(executor.submit(create_data_deb,'llm-qwen2-1.5B-prefill-20e', data_version, src_folder, revision))
337+
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision))
338+
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-1.5B-ax630c', '0.3', src_folder, revision))
339+
futures.append(executor.submit(create_data_deb,'llm-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision))
340+
futures.append(executor.submit(create_data_deb,'llm-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision))
341+
futures.append(executor.submit(create_data_deb,'llm-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision))
342+
futures.append(executor.submit(create_data_deb,'llm-internvl2.5-1B-ax630c', '0.4', src_folder, revision))
341343
futures.append(executor.submit(create_data_deb,'llm-deepseek-r1-1.5B-ax630c', '0.3', src_folder, revision))
342344
for future in concurrent.futures.as_completed(futures):
343345
result = future.result()

0 commit comments

Comments
 (0)