not really™ fix (better to change options)

Author: roj234

CMakeLists.txt

@@ -8,6 +8,25 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
+
+    message(STATUS "polyfilling #embed via Python...")
+
+    set(VOCAB_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/src/vocab/vocab.hpp")
+    set(GENERATED_VOCAB_HPP "${CMAKE_CURRENT_SOURCE_DIR}/src/vocab/vocab_generated.h")
+    execute_process(
+        COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/script/generate_vocab.py
+                ${VOCAB_HEADER}
+                ${GENERATED_VOCAB_HPP}
+        RESULT_VARIABLE result
+    )
+
+    if (NOT result EQUAL 0)
+        message(FATAL_ERROR "Failed to run generate_vocab.py")
+    endif()
+
+    add_definitions(-DUSE_GENERATED_VOCAB)
+
+
 if (MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)

script/generate_vocab.py

@@ -0,0 +1,39 @@
+import re
+import os
+import sys
+
+def file_to_c_array(filepath):
+    with open(filepath, "rb") as f:
+        data = f.read()
+
+    cont = []
+    for i, b in enumerate(data):
+        val = ""
+        if (i + 1) % 16 == 0:
+            val = "\n"
+        val += f'{b}';
+        cont.append(val)
+
+    return ",".join(cont)
+
+def process_header(input_path, output_hpp):
+    with open(input_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    pattern = re.compile(r'char\s+(\w+)\[\]\s*=?\s*\{\s*#embed\s+"([^"]+)"\s*\};')
+    matches = pattern.findall(content)
+    if not matches:
+        print("No #embed found in vocab.hpp")
+        return
+
+    out_content = f'#pragma once\n\n'
+    for var_name, file_path in matches:
+        print(f"Embedding {file_path} into {var_name}...")
+        hex_data = file_to_c_array(file_path)
+        out_content += f"static const unsigned char {var_name}[] = {{\n{hex_data}\n}};\n"
+
+    with open(output_hpp, "w") as f: f.write(out_content)
+
+if __name__ == "__main__":
+    # Usage: python embed_fix.py <vocab.hpp> <out.hpp>
+    process_header(sys.argv[1], sys.argv[2])

src/vocab/vocab.cpp

@@ -1,5 +1,8 @@
 #include "vocab.h"
 
+#ifdef USE_GENERATED_VOCAB
+#include "vocab_generated.h"
+#else
 static unsigned char clip_merges_utf8_c_str[] = {
     #embed "embed/merges.txt"
 };
@@ -18,6 +21,7 @@ static unsigned char qwen2_merges_utf8_c_str[] = {
 static unsigned char umt5_tokenizer_json_str[] = {
     #embed "embed/umt5_tokenizer.json"
 };
+#endif
 
 std::string load_clip_merges() {
     std::string merges_utf8_str(reinterpret_cast<const char*>(clip_merges_utf8_c_str), sizeof(clip_merges_utf8_c_str));