From aaf1922e55785e5d820f90191ee99c2ac711428e Mon Sep 17 00:00:00 2001 From: Alcoft Date: Fri, 3 Apr 2026 06:46:52 +0200 Subject: [PATCH 1/3] Implemented 'LFM25VLChatHandler'. --- llama_cpp/llama_chat_format.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 994a3dca0..5d9a676b1 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5084,6 +5084,72 @@ def __call__(self, **kwargs): # Use parent implementation return super().__call__(**kwargs) +class LFM25VLChatHandler(MTMDChatHandler): + CHAT_FORMAT = ( + "{{- bos_token -}}" + "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}" + "{%- set ns = namespace(system_prompt='', last_assistant_index=-1) -%}" + "{%- if messages[0]['role'] == 'system' -%}" + "{%- if messages[0]['content'] is string -%}" + "{%- set ns.system_prompt = messages[0]['content'] -%}" + "{%- else -%}" + "{%- for item in sys_content -%}" + "{%- if item['type'] == 'text' or 'text' in item -%}" + "{%- set ns.system_prompt = ns.system_prompt + item['text'] -%}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{%- set messages = messages[1:] -%}" + "{%- endif -%}" + "{%- if tools -%}" + "{%- set ns.system_prompt = ns.system_prompt ~ ('\n' if ns.system_prompt else '') ~ 'List of tools: [' -%}" + "{%- for tool in tools -%}" + "{%- set tool = (tool | tojson) if tool is not string else tool -%}" + "{%- set ns.system_prompt = ns.system_prompt ~ tool ~ (', ' if not loop.last else '') -%}" + "{%- endfor -%}" + "{%- set ns.system_prompt = ns.system_prompt ~ ']' -%}" + "{%- endif -%}" + "{{- ('<|im_start|>system\n' ~ ns.system_prompt ~ '<|im_end|>\n') if ns.system_prompt else '' -}}" + "{%- for message in messages -%}" + "{%- if message['role'] == 'assistant' -%}" + "{%- set ns.last_assistant_index = loop.index0 -%}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- for message in messages -%}" + "{{- '<|im_start|>' ~ message['role'] ~ '\n' -}}" + "{%- if message['content'] is string -%}" + "{%- set content = message['content'] -%}" + "{%- else -%}" + "{%- set content = '' -%}" + "{%- for item in message['content'] -%}" + "{%- if item['type'] in ['image', 'image_url'] and item['type'] in item -%}" + "{%- set content = content ~ (item[item['type']] if item[item['type']] is string else item[item['type']]['url']) -%}" + "{%- elif item['type'] == 'text' and 'text' in item -%}" + "{%- set content = content ~ item['text'] -%}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{%- if message['role'] == 'assistant' and not keep_past_thinking and loop.index0 != ns.last_assistant_index and '' in content -%}" + "{%- set content = content.split('')[-1] | trim -%}" + "{%- endif -%}" + "{{- content ~ '<|im_end|>\n' -}}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|im_start|>assistant\n' -}}" + "{%- if not enable_thinking -%}" + "{{- '\n\n\n' -}}" + "{%- endif -%}" + "{%- endif -%}" + ) # Variables: keep_past_thinking, enable_thinking + + def __init__(self, enable_thinking: bool = True, keep_past_thinking: bool = False, **kwargs): + super().__init__(**kwargs) + self.extra_template_arguments["enable_thinking"] = enable_thinking + self.extra_template_arguments["keep_past_thinking"] = keep_past_thinking + + def __call__(self, **kwargs): + super().__call__(**kwargs) + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama_core.Llama, From 17366d307b1ba85de5698d178a968b3d392d49e4 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Sun, 5 Apr 2026 08:59:33 +0200 Subject: [PATCH 2/3] updated code --- llama_cpp/llama_chat_format.py | 138 +++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 59 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index de164159b..a020e079b 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5414,70 +5414,90 @@ def __call__(self, **kwargs): return super().__call__(**kwargs) class LFM25VLChatHandler(MTMDChatHandler): + # Aligned with LFM2.5-VL tokenizer_config + LFM25VL_BOS_TOKEN = "<|startoftext|>" + LFM25VL_EOS_TOKEN = "<|im_end|>" + LFM25VL_PAD_TOKEN = "<|pad|>" + + # Image specific tokens + LFM25VL_IMAGE_TOKEN = "" + LFM25VL_IMAGE_START_TOKEN = "<|image_start|>" + LFM25VL_IMAGE_END_TOKEN = "<|image_end|>" + LFM25VL_IMAGE_THUMBNAIL = "<|img_thumbnail|>" + CHAT_FORMAT = ( - "{{- bos_token -}}" - "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}" - "{%- set ns = namespace(system_prompt='', last_assistant_index=-1) -%}" - "{%- if messages[0]['role'] == 'system' -%}" - "{%- if messages[0]['content'] is string -%}" - "{%- set ns.system_prompt = messages[0]['content'] -%}" - "{%- else -%}" - "{%- for item in sys_content -%}" - "{%- if item['type'] == 'text' or 'text' in item -%}" - "{%- set ns.system_prompt = ns.system_prompt + item['text'] -%}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{%- set messages = messages[1:] -%}" - "{%- endif -%}" - "{%- if tools -%}" - "{%- set ns.system_prompt = ns.system_prompt ~ ('\n' if ns.system_prompt else '') ~ 'List of tools: [' -%}" - "{%- for tool in tools -%}" - "{%- set tool = (tool | tojson) if tool is not string else tool -%}" - "{%- set ns.system_prompt = ns.system_prompt ~ tool ~ (', ' if not loop.last else '') -%}" - "{%- endfor -%}" - "{%- set ns.system_prompt = ns.system_prompt ~ ']' -%}" - "{%- endif -%}" - "{{- ('<|im_start|>system\n' ~ ns.system_prompt ~ '<|im_end|>\n') if ns.system_prompt else '' -}}" - "{%- for message in messages -%}" - "{%- if message['role'] == 'assistant' -%}" - "{%- set ns.last_assistant_index = loop.index0 -%}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- for message in messages -%}" - "{{- '<|im_start|>' ~ message['role'] ~ '\n' -}}" - "{%- if message['content'] is string -%}" - "{%- set content = message['content'] -%}" - "{%- else -%}" - "{%- set content = '' -%}" - "{%- for item in message['content'] -%}" - "{%- if item['type'] in ['image', 'image_url'] and item['type'] in item -%}" - "{%- set content = content ~ (item[item['type']] if item[item['type']] is string else item[item['type']]['url']) -%}" - "{%- elif item['type'] == 'text' and 'text' in item -%}" - "{%- set content = content ~ item['text'] -%}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{%- if message['role'] == 'assistant' and not keep_past_thinking and loop.index0 != ns.last_assistant_index and '' in content -%}" - "{%- set content = content.split('')[-1] | trim -%}" - "{%- endif -%}" - "{{- content ~ '<|im_end|>\n' -}}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - "{{- '<|im_start|>assistant\n' -}}" - "{%- if not enable_thinking -%}" - "{{- '\n\n\n' -}}" - "{%- endif -%}" - "{%- endif -%}" - ) # Variables: keep_past_thinking, enable_thinking + "{{- bos_token -}}\n" + "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n" + "{%- set ns = namespace(system_prompt='', content='') -%}\n" + "{%- if messages[0]['role'] == 'system' -%}\n" + " {%- set ns.system_prompt = messages[0]['content'] -%}\n" + " {%- set messages = messages[1:] -%}\n" + "{%- endif -%}\n" + "{%- if tools -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ('\\n' if ns.system_prompt else '') + 'List of tools: [' -%}\n" + " {%- for tool in tools -%}\n" + " {%- if tool is not string -%}\n" + " {%- set tool = tool | tojson -%}\n" + " {%- endif -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + tool -%}\n" + " {%- if not loop.last -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ', ' -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ']' -%}\n" + "{%- endif -%}\n" + "{%- if ns.system_prompt -%}\n" + " {{- '<|im_start|>system\\n' + ns.system_prompt + '<|im_end|>\\n' -}}\n" + "{%- endif -%}\n" + "{%- set ns.last_assistant_index = -1 -%}\n" + "{%- for message in messages -%}\n" + " {%- if message['role'] == 'assistant' -%}\n" + " {%- set ns.last_assistant_index = loop.index0 -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "{%- for message in messages -%}\n" + " {{- '<|im_start|>' + message['role'] + '\\n' -}}\n" + " {%- set content = message['content'] -%}\n" + " {%- if content is not string -%}\n" + " {%- set ns.content = '' -%}\n" + " {#- MTMD-style Multimodal Injection (Audio stripped for VL model) -#}\n" + " {%- for item in content -%}\n" + " {%- if item['type'] == 'image_url' -%}\n" + " {%- set img_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {%- set ns.content = ns.content + img_val -%}\n" + " {%- elif item['type'] == 'text' -%}\n" + " {%- set ns.content = ns.content + item['text'] -%}\n" + " {%- else -%}\n" + " {%- set ns.content = ns.content + (item | tojson) -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- set content = ns.content -%}\n" + " {%- endif -%}\n" + " {%- if message['role'] == 'assistant' and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n" + " {%- if '' in content -%}\n" + " {%- set content = content.split('')[-1] | trim -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {{- content + '<|im_end|>\\n' -}}\n" + "{%- endfor -%}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) - def __init__(self, enable_thinking: bool = True, keep_past_thinking: bool = False, **kwargs): + def __init__(self, keep_past_thinking: bool = False, **kwargs): + self.keep_past_thinking = keep_past_thinking super().__init__(**kwargs) - self.extra_template_arguments["enable_thinking"] = enable_thinking - self.extra_template_arguments["keep_past_thinking"] = keep_past_thinking + def __call__(self, **kwargs): - super().__call__(**kwargs) + self.extra_template_arguments["keep_past_thinking"] = self.keep_past_thinking + + kwargs['stop'] = [self.LFM25VL_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(keep_past_thinking={self.keep_past_thinking}) - Start processing") + return super().__call__(**kwargs) @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( From 00e67b8f527b3c5bc63131414b300408f78700a1 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Mon, 6 Apr 2026 03:18:49 +0200 Subject: [PATCH 3/3] prevent errors by setting 'image_min_tokens' to 256 if higher values are detected --- llama_cpp/llama_chat_format.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index a020e079b..91e3ad699 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5491,6 +5491,12 @@ def __init__(self, keep_past_thinking: bool = False, **kwargs): def __call__(self, **kwargs): + if self.image_min_tokens > 256: + if self.verbose: + print(f"For LFM2.5-VL, using values higher than 256 for `image_min_tokens` could cause errors. Setting to **256**.") + + self.image_min_tokens = 256 + self.extra_template_arguments["keep_past_thinking"] = self.keep_past_thinking kwargs['stop'] = [self.LFM25VL_EOS_TOKEN]