Add Qwen2 model classes

Author: CodeWithKyrian

docs/getting-started.md

@@ -12,6 +12,7 @@ Before installing TransformersPHP, ensure your system meets the following requir
 - Composer
 - PHP FFI extension
 - JIT compilation (optional)
+- Increased memory limit (for advanced tasks like text generation)
 
 ## Installation
 

examples/pipelines/text-generation.php

@@ -11,7 +11,7 @@
 
 ini_set('memory_limit', -1);
 //
-$generator = pipeline('text-generation', 'Xenova/gpt2');
+$generator = pipeline('text-generation', 'Xenova/Qwen1.5-0.5B-Chat');
 
 $streamer = StdOutStreamer::make($generator->tokenizer);
 
@@ -21,7 +21,9 @@
     ['role' => 'user', 'content' => 'I am doing great. What about you?'],
 ];
 
-$output = $generator("I love going to school but I don't",
+$input = $generator->tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false);
+
+$output = $generator($messages,
     streamer: $streamer,
     maxNewTokens: 128,
     doSample: true,

src/Models/Auto/AutoModel.php

@@ -38,6 +38,7 @@ class AutoModel extends PretrainedMixin
         "gptj" => \Codewithkyrian\Transformers\Models\Pretrained\GPTJModel::class,
         "gpt_bigcode" => \Codewithkyrian\Transformers\Models\Pretrained\GPTBigCodeModel::class,
         "codegen" => \Codewithkyrian\Transformers\Models\Pretrained\CodeGenModel::class,
+        "qwen2" => \Codewithkyrian\Transformers\Models\Pretrained\Qwen2Model::class,
     ];
 
     const MODEL_CLASS_MAPPINGS = [

src/Models/Auto/AutoModelForCausalLM.php

@@ -12,8 +12,7 @@ class AutoModelForCausalLM extends PretrainedMixin
         'gptj' => \Codewithkyrian\Transformers\Models\Pretrained\GPTJForCausalLM::class,
         'gpt_bigcode' => \Codewithkyrian\Transformers\Models\Pretrained\GPTBigCodeForCausalLM::class,
         'codegen' => \Codewithkyrian\Transformers\Models\Pretrained\CodeGenForCausalLM::class,
-        'trocr' => \Codewithkyrian\Transformers\Models\Pretrained\TrOCRForCausalLM::class,
-
+        'qwen2' => \Codewithkyrian\Transformers\Models\Pretrained\Qwen2ForCausalLM::class
     ];
 
     const MODEL_CLASS_MAPPINGS = [

src/Models/Pretrained/Qwen2ForCausalLM.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+class Qwen2ForCausalLM extends Qwen2PreTrainedModel
+{
+
+}

src/Models/Pretrained/Qwen2Model.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+class Qwen2Model extends Qwen2PreTrainedModel
+{
+
+}

src/Models/Pretrained/Qwen2PreTrainedModel.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+use Codewithkyrian\Transformers\Models\ModelArchitecture;
+use Codewithkyrian\Transformers\Utils\AutoConfig;
+use Codewithkyrian\Transformers\Utils\GenerationConfig;
+use OnnxRuntime\InferenceSession;
+
+/**
+ * The bare Qwen2 Model outputting raw hidden-states without any specific head on top.
+ */
+class Qwen2PreTrainedModel extends PreTrainedModel
+{
+    protected int $numHeads;
+    protected int $numLayers;
+    protected int $dimKv;
+
+    public function __construct(
+        AutoConfig               $config,
+        InferenceSession         $session,
+        public ModelArchitecture $modelArchitecture,
+        public GenerationConfig  $generationConfig
+    )
+    {
+        parent::__construct($config, $session, $modelArchitecture);
+
+        // config doesn't contain pad_token_id, so we assume it is the eos_token_id
+        $this->config['pad_token_id'] = $this->config['eos_token_id'];
+        $this->config->padTokenId = $this->config['eos_token_id'];
+
+        $this->numHeads = $this->config['num_key_value_heads'] ??  $this->config['num_attention_heads'];
+        $this->numLayers = $this->config['num_hidden_layers'];
+        $this->dimKv = $this->config['hidden_size'] / $this->config['num_attention_heads'];
+    }
+}

src/PreTokenizers/SplitPreTokenizer.php

@@ -23,14 +23,15 @@ public function preTokenizeText(string|array $text, array $options): array
             return explode($this->pattern, $text);
         }
 
-        $regex = $this->pattern['Regex'] ?? $this->pattern['String'] ?? null;
+        $regex = $this->pattern['Regex'] ?? null;
 
-        if ($regex != null) {
-            preg_match_all("/$regex/u", $text, $matches, PREG_SPLIT_NO_EMPTY);
-
-            $text = array_map(fn($match) => $match, $matches[0]);
+        if($regex != null)
+        {
+           $split =  preg_split($regex, $text, -1, PREG_SPLIT_NO_EMPTY);
+           dd($split);
         }
 
+        // TODO: Handle all types of Regex
         return $text;
     }
 }

src/PretrainedTokenizers/AutoTokenizer.php

@@ -55,6 +55,7 @@ class AutoTokenizer
         'SpeechT5Tokenizer' => 'SpeechT5Tokenizer',
         'NougatTokenizer' => 'NougatTokenizer',
         'VitsTokenizer' => 'VitsTokenizer',
+        'Qwen2Tokenizer' => 'Qwen2Tokenizer',
         // Base case:
         'PreTrainedTokenizer' => 'PreTrainedTokenizer',
     ];

src/PretrainedTokenizers/Qwen2Tokenizer.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\PretrainedTokenizers;
+
+class Qwen2Tokenizer extends PretrainedTokenizer
+{
+
+}