Add a simple(but limited) jinja template parser

Author: CodeWithKyrian

examples/tokenizers/apply-chat-template.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+use Codewithkyrian\Transformers\PretrainedTokenizers\AutoTokenizer;
+
+require_once './bootstrap.php';
+
+//$tokenizer = AutoTokenizer::fromPretrained('mistralai/Mistral-7B-Instruct-v0.1');
+$tokenizer = AutoTokenizer::fromPretrained('facebook/blenderbot-400M-distill');
+$messages = [
+    ['role' => 'user', 'content' => 'Hello!'],
+    ['role' => 'assistant', 'content' => 'Hi! How are you?'],
+    ['role' => 'user', 'content' => 'I am doing great.'],
+    ['role' => 'assistant', 'content' => 'That is great to hear.'],
+];
+
+$text = $tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false);
+
+dd($text);

src/Exceptions/TemplateParseException.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Exceptions;
+
+class TemplateParseException extends \Exception implements TransformersException
+{
+    public static function undefinedVariable($variableName): TemplateParseException
+    {
+        return new self("Undefined variable:  $variableName");
+    }
+}

src/PretrainedTokenizers/PretrainedTokenizer.php

@@ -5,20 +5,23 @@
 
 namespace Codewithkyrian\Transformers\PretrainedTokenizers;
 
+use ByJG\JinjaPhp\Template;
+use ByJG\JinjaPhp\Undefined\DebugUndefined;
 use Codewithkyrian\Transformers\Decoders\Decoder;
 use Codewithkyrian\Transformers\Normalizers\Normalizer;
 use Codewithkyrian\Transformers\PostProcessors\PostProcessedOutput;
 use Codewithkyrian\Transformers\PostProcessors\PostProcessor;
 use Codewithkyrian\Transformers\PreTokenizers\PreTokenizer;
 use Codewithkyrian\Transformers\Tokenizers\AddedToken;
 use Codewithkyrian\Transformers\Tokenizers\Tokenizer;
+use Codewithkyrian\Transformers\Utils\JinjaTemplate;
 use Codewithkyrian\Transformers\Utils\Tensor;
 
 class PretrainedTokenizer
 {
     protected bool $returnTokenTypeIds = false;
 
-    protected bool $warnedAboutChatTemplate;
+    protected bool $warnedAboutChatTemplate = false;
 
     protected string $defaultChatTemplate = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}";
 
@@ -68,7 +71,7 @@ class PretrainedTokenizer
     protected bool $legacy;
 
     protected mixed $chatTemplate;
-    protected \SplObjectStorage $compiledTemplateCache;
+    protected array $compiledTemplateCache = [];
 
     /**
      * @param array $tokenizerJSON The JSON of the tokenizer.
@@ -145,8 +148,6 @@ public function __construct(protected array $tokenizerJSON, protected array $tok
         $this->legacy = false;
 
         $this->chatTemplate = $tokenizerConfig['chat_template'] ?? null;
-        $this->compiledTemplateCache = new \SplObjectStorage();
-
     }
 
     /**
@@ -335,7 +336,7 @@ function ($key) {
             if (
                 array_reduce($encodedTokens, function ($carry, $x) use ($encodedTokens) {
                     foreach ($x as $key => $value) {
-                        if (count($value) !== count($encodedTokens[0][$key] ?? [])) {
+                        if (count($value ?? []) !== count($encodedTokens[0][$key] ?? [])) {
                             return true;
                         }
                     }
@@ -610,10 +611,10 @@ function truncateHelper(array &$item, int $length): void
 
     protected function getDefaultChatTemplate(): string
     {
-        if (!$this->warnedAboutChatTemplate) {
-            trigger_error("The default chat template is deprecated and will be removed in a future version. Please use the `chat_template` option instead.", E_USER_WARNING);
-            $this->warnedAboutChatTemplate = true;
-        }
+//        if (!$this->warnedAboutChatTemplate) {
+//            trigger_error("The default chat template is deprecated and will be removed in a future version. Please use the `chat_template` option instead.", E_USER_WARNING);
+//            $this->warnedAboutChatTemplate = true;
+//        }
 
         return $this->defaultChatTemplate;
     }
@@ -670,7 +671,8 @@ public function applyChatTemplate(
 
         if ($compiledTemplate === null) {
             // TODO: Use Jinja to compile the template
-            $compiledTemplate = null;
+            $compiledTemplate =  new JinjaTemplate($chatTemplate);
+//            $compiledTemplate->withUndefined(new DebugUndefined());
             $this->compiledTemplateCache[$chatTemplate] = $compiledTemplate;
         }
 
@@ -682,7 +684,6 @@ public function applyChatTemplate(
             }
         }
 
-
         $rendered = $compiledTemplate->render(array_merge([
             'messages' => $conversation,
             'add_generation_prompt' => $addGenerationPrompt,
@@ -695,10 +696,10 @@ public function applyChatTemplate(
                 addSpecialTokens: false,
                 truncation: $truncation,
                 maxLength: $maxLength
-            )['input_ids'];
+            )['input_ids']->toArray();
         }
 
-        return $rendered;
+        return stripcslashes($rendered);
     }
 
 }