feat: refactor model generation for simplicity and efficiency

- New Stopping Criteria: Added MaxLength, MaxTime, and Interruptable stopping criteria for more flexible generation control.
- Streamers Refactor: Simplified streamer implementation to improve clarity.
- Performance Benchmarking: Introduced a token-per-second (TPS) metric to benchmark model performance across updates.
- Bug Fixes:
  - Fixed an error in the Tensor::slice() method
  - Corrected the RepetitionPenaltyLogitsProcessor to properly utilize tokens.

Author: CodeWithKyrian

examples/pipelines/summarization.php

@@ -58,4 +58,4 @@
 
 $summary = $summarizer($article, streamer: $streamer, maxNewTokens: 512);
 
-dd("Done", timeUsage(), memoryUsage());
+dd("Done", timeUsage(), memoryUsage());

examples/pipelines/text-generation.php

@@ -47,4 +47,4 @@
 //    returnFullText: true,
 //);
 
-dd($output[0]['generated_text'], timeUsage(), memoryUsage());
+dd($output[0]['generated_text'], $streamer->getTPS()." tps", timeUsage(), memoryUsage());

examples/pipelines/text2text-generation.php

@@ -21,5 +21,4 @@
 
 $output = $generator($query, streamer: $streamer, maxNewTokens: 256, doSample: true, repetitionPenalty: 1.1, temperature: 0.7);
 
-//dd($output);
-dd('Done', timeUsage(), memoryUsage());
+dd('Done', $streamer->getTPS()." tps", timeUsage(), memoryUsage());

src/Generation/LogitsProcessors/LogitsProcessorList.php

@@ -28,7 +28,7 @@ public function push(LogitsProcessor $item): void
      *
      * @param LogitsProcessor[] $items The logits processor functions to add.
      */
-    public function extend(traversable $items): void
+    public function extend(Traversable $items): void
     {
         foreach ($items as $item) {
             $this->processors[] = $item;
@@ -41,13 +41,15 @@ public function extend(traversable $items): void
      * @param array $inputIds The input IDs for the language model.
      * @param Tensor $batchedLogits A 2D array of logits, where each row corresponds to a single input sequence.
      */
-    public function __invoke(array $inputIds, Tensor &$batchedLogits): void
+    public function __invoke(array $inputIds, Tensor &$batchedLogits): Tensor
     {
-        for ($i = 0; $i < count($batchedLogits); $i++) {
-            foreach ($this->processors as $processor) {
-                $processor($inputIds, $batchedLogits[$i]); // Apply processors in-place
-            }
+        $toReturn = $batchedLogits;
+
+        foreach ($this->processors as $processor) {
+            $toReturn = $processor($inputIds, $toReturn); // Some apply processors in-place
         }
+
+        return $toReturn;
     }
 
     /**
@@ -59,4 +61,4 @@ public function getIterator(): Traversable
     {
         yield from $this->processors;
     }
-}
+}

src/Generation/LogitsProcessors/RepetitionPenaltyLogitsProcessor.php

@@ -12,9 +12,7 @@
  */
 class RepetitionPenaltyLogitsProcessor extends LogitsProcessor
 {
-    public function __construct(protected float $penalty)
-    {
-    }
+    public function __construct(protected float $penalty) {}
 
     /**
      * Apply the repetition penalty to the logits.
@@ -24,13 +22,16 @@ public function __invoke(array $inputIds, Tensor $logits): Tensor
         // Modify the logits corresponding to each element in `input_ids`.
         // As a consequence, the logits corresponding to tokens that appear
         // many times in the output will be penalised more.
-        foreach ($inputIds as $inputId) {
-            if ($logits->buffer()[$inputId] < 0) {
-                $logits->buffer()[$inputId] *= $this->penalty;
-            } else {
-                $logits->buffer()[$inputId] /= $this->penalty;
+        for ($i = 0; $i < count($inputIds); $i++) {
+            foreach ($inputIds[$i] as $inputId) {
+                if ($logits[$i]->buffer()[$inputId] < 0) {
+                    $logits[$i]->buffer()[$inputId] *= $this->penalty;
+                } else {
+                    $logits[$i]->buffer()[$inputId] /= $this->penalty;
+                }
             }
         }
+
         return $logits;
     }
-}
+}

src/Generation/StoppingCriteria/EosTokenCriteria.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Transformers\Generation\StoppingCriteria;
+
+/**
+ * This class stops generation whenever the "end-of-sequence" token is generated.
+ */
+class EosTokenCriteria extends StoppingCriteria
+{
+    private array $eosTokenIds;
+
+    /**
+     * @param int|int[] $eosTokenId The id of the *end-of-sequence* token.
+     */
+    public function __construct(int|array $eosTokenId)
+    {
+        $this->eosTokenIds = is_array($eosTokenId) ? $eosTokenId : [$eosTokenId];
+    }
+
+    public function __invoke(array $inputIds, array $scores): array
+    {
+        return array_map(function ($ids) {
+            $lastToken = end($ids);
+            return in_array($lastToken, $this->eosTokenIds, true);
+        }, $inputIds);
+    }
+}

src/Generation/StoppingCriteria/InterruptableStoppingCriteria.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Transformers\Generation\StoppingCriteria;
+
+/**
+ * This class stops generation whenever the user interrupts the process.
+ */
+class InterruptableStoppingCriteria extends StoppingCriteria
+{
+    private bool $interrupted = false;
+
+    public function interrupt(): void
+    {
+        $this->interrupted = true;
+    }
+
+    public function reset(): void
+    {
+        $this->interrupted = false;
+    }
+
+    public function __invoke(array $inputIds, array $scores): array
+    {
+        return array_fill(0, count($inputIds), $this->interrupted);
+    }
+}

src/Generation/StoppingCriteria/MaxLengthCriteria.php

@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Transformers\Generation\StoppingCriteria;
+
+use Codewithkyrian\Transformers\Transformers;
+
+/**
+ * This class stops generation whenever the full generated number of tokens exceeds `maxLength`.
+ * For decoder-only transformers, this includes the initial prompted tokens.
+ */
+class MaxLengthCriteria extends StoppingCriteria
+{
+
+
+    /**
+     * @param int $maxLength The maximum length that the output sequence can have in number of tokens.
+     * @param int|null $maxPositionEmbeddings The maximum model length,
+     */
+    public function __construct(protected int $maxLength, protected ?int $maxPositionEmbeddings = null) {}
+
+    /**
+     * Evaluates whether generation should stop based on token count.
+     *
+     * @param array $inputIds Array of input IDs (2D array where each sub-array is a sequence of token IDs).
+     * @param array $scores Optional scores for the generated tokens.
+     *
+     * @return array|bool[]
+     */
+    public function __invoke(array $inputIds, array $scores): array
+    {
+//        return array_map(fn ($ids) => count($ids) >= $this->maxLength, $inputIds);
+        $results = [];
+        foreach ($inputIds as $ids) {
+            $currentLength = count($ids);
+            $isDone = $currentLength >= $this->maxLength;
+
+            if ($this->maxPositionEmbeddings !== null && !$isDone && $currentLength >= $this->maxPositionEmbeddings) {
+                echo
+                    "This is a friendly reminder - the current text generation call will exceed the model's predefined " .
+                    "maximum length ({$this->maxPositionEmbeddings}). Depending on the model, you may observe " .
+                    "exceptions, performance degradation, or nothing at all."
+                ;
+            }
+
+            $results[] = $isDone;
+        }
+
+        return $results;
+    }
+}

src/Generation/StoppingCriteria/MaxTimeCriteria.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Transformers\Generation\StoppingCriteria;
+
+class MaxTimeCriteria extends StoppingCriteria
+{
+    private float $maxTime;
+    private float $initialTimestamp;
+
+    /**
+     * @param float $maxTime The maximum allowed time in seconds for the generation.
+     * @param float|null $initialTimestamp The start of the generation allowed time. Defaults to the current time.
+     */
+    public function __construct(float $maxTime, ?float $initialTimestamp = null)
+    {
+        $this->maxTime = $maxTime;
+        $this->initialTimestamp = $initialTimestamp ?? microtime(true);
+    }
+
+    /**
+     * Evaluates whether generation should stop based on elapsed time.
+     *
+     * @param array $inputIds Array of input IDs (2D array where each sub-array is a sequence of token IDs).
+     * @param array $scores  Scores for the generated tokens.
+     *
+     * @return array Boolean array indicating whether generation should stop for each sequence.
+     */
+    public function __invoke(array $inputIds, array $scores): array
+    {
+        $elapsedTime = microtime(true) - $this->initialTimestamp;
+        $isDone = $elapsedTime > $this->maxTime;
+
+        // Return the same stopping criteria for all sequences
+        return array_fill(0, count($inputIds), $isDone);
+    }
+}

src/Generation/StoppingCriteria/StoppingCriteria.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codewithkyrian\Transformers\Generation\StoppingCriteria;
+
+use Traversable;
+
+/**
+ * Abstract base class for all stopping criteria that can be applied during generation.
+ */
+abstract class StoppingCriteria
+{
+    /**
+     * @param int[][] $inputIds Indices of input sequence tokens in the vocabulary of shape `(batch_size, sequence_length)`.
+     * @param float[][] $scores Prediction scores of a language modeling head of shape `(batch_size, vocab_size)`.
+     *
+     * @return bool[] A list of booleans indicating whether each sequence should be stopped.
+     */
+    abstract public function __invoke(array $inputIds, array $scores): array;
+}