feat: Use the new PHP Jinja template engine

Author: CodeWithKyrian

composer.json

@@ -18,7 +18,8 @@
     "ankane/onnxruntime": "^0.2.0",
     "ext-gd": "*",
     "rindow/rindow-math-matrix": "^1.2",
-    "spatie/fork": "^1.2"
+    "spatie/fork": "^1.2",
+    "codewithkyrian/jinja-php": "^1.0"
   },
   "require-dev": {
     "pestphp/pest": "^2.31",

examples/tokenizers/apply-chat-template.php

@@ -6,8 +6,8 @@
 
 require_once './bootstrap.php';
 
-//$tokenizer = AutoTokenizer::fromPretrained('mistralai/Mistral-7B-Instruct-v0.1');
-$tokenizer = AutoTokenizer::fromPretrained('facebook/blenderbot-400M-distill');
+$tokenizer = AutoTokenizer::fromPretrained('mistralai/Mistral-7B-Instruct-v0.1');
+//$tokenizer = AutoTokenizer::fromPretrained('facebook/blenderbot-400M-distill');
 $messages = [
     ['role' => 'user', 'content' => 'Hello!'],
     ['role' => 'assistant', 'content' => 'Hi! How are you?'],

src/Pipelines/TextGenerationPipeline.php

@@ -67,6 +67,7 @@ public function __invoke(array|string $texts, ...$args): array
         $generationConfig = new GenerationConfig($snakeCasedArgs);
 
         $isBatched = is_array($texts);
+
         if (!$isBatched) {
             $texts = [$texts];
         }

src/PretrainedTokenizers/PretrainedTokenizer.php

@@ -5,16 +5,14 @@
 
 namespace Codewithkyrian\Transformers\PretrainedTokenizers;
 
-use ByJG\JinjaPhp\Template;
-use ByJG\JinjaPhp\Undefined\DebugUndefined;
+use Codewithkyrian\Jinja\Template;
 use Codewithkyrian\Transformers\Decoders\Decoder;
 use Codewithkyrian\Transformers\Normalizers\Normalizer;
 use Codewithkyrian\Transformers\PostProcessors\PostProcessedOutput;
 use Codewithkyrian\Transformers\PostProcessors\PostProcessor;
 use Codewithkyrian\Transformers\PreTokenizers\PreTokenizer;
 use Codewithkyrian\Transformers\Tokenizers\AddedToken;
 use Codewithkyrian\Transformers\Tokenizers\Tokenizer;
-use Codewithkyrian\Transformers\Utils\JinjaTemplate;
 use Codewithkyrian\Transformers\Utils\Tensor;
 
 class PretrainedTokenizer
@@ -671,8 +669,7 @@ public function applyChatTemplate(
 
         if ($compiledTemplate === null) {
             // TODO: Use Jinja to compile the template
-            $compiledTemplate =  new JinjaTemplate($chatTemplate);
-//            $compiledTemplate->withUndefined(new DebugUndefined());
+            $compiledTemplate =  new Template($chatTemplate);
             $this->compiledTemplateCache[$chatTemplate] = $compiledTemplate;
         }
 
@@ -684,6 +681,7 @@ public function applyChatTemplate(
             }
         }
 
+
         $rendered = $compiledTemplate->render(array_merge([
             'messages' => $conversation,
             'add_generation_prompt' => $addGenerationPrompt,