New tensor mehods - topk, divide, slice, and change implementations to use these methods

CodeWithKyrian · CodeWithKyrian · commit dda9a70bfd57 · 2024-05-01T09:06:51.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ playground/*
 .idea
 .transformers-cache/*
 tests/models/*
+dist
diff --git a/composer.json b/composer.json
@@ -14,15 +14,16 @@
   "type": "library",
   "require": {
     "php": "^8.1",
-    "guzzlehttp/guzzle": "^7.0",
     "ankane/onnxruntime": "^0.2.0",
-    "rindow/rindow-math-matrix": "^2.0",
+    "guzzlehttp/guzzle": "^7.0",
     "codewithkyrian/jinja-php": "^1.0",
     "codewithkyrian/onnxruntime-downloader-plugin": "^1.1",
     "symfony/console": "^6.4|^7.0",
     "imagine/imagine": "^1.3",
     "rokka/imagine-vips": "^0.31.0",
-    "rindow/rindow-math-matrix-matlibffi": "^1.0"
+    "rindow/rindow-math-matrix": "^2.0",
+    "rindow/rindow-matlib-ffi": "^1.0",
+    "rindow/rindow-openblas-ffi": "^1.0"
   },
   "require-dev": {
     "pestphp/pest": "^2.31",
diff --git a/examples/pipelines/text-generation.php b/examples/pipelines/text-generation.php
@@ -12,8 +12,8 @@
 ini_set('memory_limit', -1);
 //
 //$generator = pipeline('text-generation', 'Xenova/gpt2');
-$generator = pipeline('text-generation', 'Xenova/Qwen1.5-0.5B-Chat');
-//$generator = pipeline('text-generation', 'Xenova/TinyLlama-1.1B-Chat-v1.0');
+//$generator = pipeline('text-generation', 'Xenova/Qwen1.5-0.5B-Chat');
+$generator = pipeline('text-generation', 'Xenova/TinyLlama-1.1B-Chat-v1.0');
 
 $streamer = StdOutStreamer::make();
 
diff --git a/libs/.gitignore b/libs/.gitignore
@@ -0,0 +1 @@
+/*
diff --git a/libs/openblas-osx-arm64-0.3.27/include/openblas.h b/libs/openblas-osx-arm64-0.3.27/include/openblas.h
diff --git a/src/FeatureExtractors/ImageFeatureExtractor.php b/src/FeatureExtractors/ImageFeatureExtractor.php
@@ -453,7 +453,6 @@ public function preprocess(
 
         $reshapedInputSize = [$image->height(), $image->width()];
 
-
         // All pixel-level manipulation occurs with data in the hwc format (height, width, channels),
         // to emulate the behavior of the original Python code (w/ numpy).
         $pixelData = $image->pixelData();
@@ -545,7 +544,6 @@ public function __invoke(Image|array $images, ...$args): array
             $originalSizes[] = $data['original_size'];
             $reshapedInputSizes[] = $data['reshaped_input_size'];
         }
-
         return [
             'pixel_values' => $stackedPixelValues,
             'original_sizes' => $originalSizes,
diff --git a/src/Generation/Samplers/BeamSearchSampler.php b/src/Generation/Samplers/BeamSearchSampler.php
@@ -18,28 +18,27 @@ class BeamSearchSampler extends Sampler
      * @param int $index
      * @return array
      */
-    public function sample(Tensor $logits, int $index)
+    public function sample(Tensor $logits, int $index): array
     {
-        $shape = $logits->shape();
-        $k = end($shape); // defaults to vocab size
+        $vocabSize = $logits->shape()[$logits->ndim() - 1];
 
-        if ($this->generationConfig->top_k > 0) {
-            $k = min($this->generationConfig->top_k, $k);
-        }
+        $k = $this->generationConfig->top_k > 0
+            ? min($this->generationConfig->top_k, $vocabSize)
+            : $vocabSize; // defaults to vocab size
 
         // Get logits of nth token
         $logs = $this->getLogits($logits, $index);
 
         // Get top k tokens
-        $topLogits = Math::getTopItems($logs, $k);
+        [$topLogits, $topIndices] = $logs->topk($k);
 
         // Compute softmax over logits
-        $probabilities = Math::softmax(array_column($topLogits, 1));
+        $probabilities = $topLogits->softmax()->toArray();
 
         $sampledResults = [];
         for ($i = 0; $i < $this->generationConfig->num_beams; $i++) {
             $sampledResults[] = [
-                $topLogits[$i][0], // token id
+                $topIndices[$i], // token id
                 log($probabilities[$i]), // score
             ];
         }
diff --git a/src/Generation/Samplers/GreedySampler.php b/src/Generation/Samplers/GreedySampler.php
@@ -21,12 +21,11 @@ public function sample(Tensor $logits, int $index): array
     {
         // NOTE: no need to do log_softmax here since we only take the maximum
         $logs = $this->getLogits($logits, $index);
-        $argmax = array_search(max($logs), $logs);
 
         // Note: score is meaningless in this context, since we are performing
         // greedy search (p = 1 => log(p) = 0)
         return [
-            [$argmax, 0]
+            [$logs->argMax(), 0]
         ];
     }
 }
diff --git a/src/Generation/Samplers/MultinomialSampler.php b/src/Generation/Samplers/MultinomialSampler.php
@@ -7,6 +7,7 @@
 
 use Codewithkyrian\Transformers\Utils\Math;
 use Codewithkyrian\Transformers\Utils\Tensor;
+use function Codewithkyrian\Transformers\Utils\timeUsage;
 
 class MultinomialSampler extends Sampler
 {
@@ -17,31 +18,30 @@ class MultinomialSampler extends Sampler
      * @param int $index
      * @return array
      */
-    public function sample(Tensor $logits, int $index)
+    public function sample(Tensor $logits, int $index): array
     {
-        $shape = $logits->shape();
-        $k = end($shape); // defaults to vocab size
+        $vocabSize = $logits->shape()[$logits->ndim() - 1];
 
-        if ($this->generationConfig->top_k > 0) {
-            $k = min($this->generationConfig->top_k, $k);
-        }
+        $k = $this->generationConfig->top_k > 0
+            ? min($this->generationConfig->top_k, $vocabSize)
+            : $vocabSize; // defaults to vocab size
 
         // Get logits of nth token
         $logs = $this->getLogits($logits, $index);
 
         // Get top k tokens
-        $topLogits = Math::getTopItems($logs, $k);
+        [$topLogits, $topIndices] = $logs->topk($k);
 
         // Compute softmax over logits
-        $probabilities = Math::softmax(array_column($topLogits, 1));
-
+        $probabilities = $topLogits->softmax()->toArray();
 
         $sampledResults = [];
+
         for ($i = 0; $i < $this->generationConfig->num_beams; $i++) {
             $sampledIndex = $this->randomSelect($probabilities);
 
             $sampledResults[] = [
-                $topLogits[$sampledIndex][0], // token id
+                $topIndices[$sampledIndex], // token id
                 log($probabilities[$sampledIndex]), // score
             ];
         }
diff --git a/src/Generation/Samplers/Sampler.php b/src/Generation/Samplers/Sampler.php
@@ -44,9 +44,10 @@ abstract public function sample(Tensor $logits, int $index);
      * @param int $index
      * @return array
      */
-    public function getLogits(Tensor $logits, int $index): array
+    public function getLogits(Tensor $logits, int $index): Tensor
     {
-        $vocabSize = $logits->shape()[count($logits->shape()) - 1];
+        $vocabSize = $logits->shape()[$logits->ndim() - 1];
+
 //        $logs = $logits->buffer()->toArray();
 //
 //        if ($index === -1) {
@@ -56,21 +57,23 @@ public function getLogits(Tensor $logits, int $index): array
 //            $logs = array_slice($logs, $startIndex, $startIndex + $vocabSize);
 //        }
 
-        $start = $index === -1 ? $logits->buffer()->count() - $vocabSize : $index * $vocabSize;
-        $end = $start + $vocabSize;
+        $start = array_fill(0, $logits->ndim() - 2, 0);
+        $size = array_fill(0, $logits->ndim() - 2, 1);
 
-        $logs = [];
+        $start[] = $index;
+        $size[] = 1;
 
-        for ($i = $start; $i < $end; $i++) {
-            $logs[] = $logits->buffer()[$i];
-        }
+        $start[] = -$vocabSize;
+        $size[] = $vocabSize;
+
+        $logs = $logits->newSlice($start, $size);
 
-        // add temperature
         if ($this->generationConfig->temperature > 0) {
-            $logs = array_map(fn($x) => $x / $this->generationConfig->temperature, $logs);
+            $logs = $logs->divide($this->generationConfig->temperature);
         }
 
-        return $logs;
+        // Remove all dimensions of 1, leaving a flat 1D array of vocab_size
+        return $logs->squeeze();
     }
 
     /**
@@ -85,6 +88,7 @@ public function randomSelect(array $probabilities): int
 
         // Generate a random number between 0 and the sum of probabilities
         $r = mt_rand() / mt_getrandmax() * $sumProbabilities;
+
         foreach ($probabilities as $i => $probability) {
             $r -= $probability;
 
diff --git a/src/Models/Pretrained/PretrainedModel.php b/src/Models/Pretrained/PretrainedModel.php
@@ -625,7 +625,7 @@ public function generate(
                 // In most cases, this will be [batch_size, 1, vocab_size]
                 // So, we select the last token's logits:
                 // (equivalent to `logits = outputs.logits[:, -1, :]`)
-                $logits = $output['logits']->slice(null, -1, null);
+                $logits = $output['logits']-lice(null, -1, null);
 
                 // Apply logits processor
                 $logitsProcessor($beam['output_token_ids'], $logits);
diff --git a/src/Pipelines/FillMaskPipeline.php b/src/Pipelines/FillMaskPipeline.php
@@ -51,22 +51,23 @@ public function __invoke(array|string $inputs, ...$args): array
                 throw new \Error("Mask token ({$this->tokenizer->maskToken}) not found in text.");
             }
 
-            $logits = $outputs->logits[$i]->toArray();
-            $itemLogits = $logits[$maskTokenIndex];
+            $logits = $outputs->logits[$i][$maskTokenIndex];
 
-            $scores = Math::getTopItems(Math::softmax($itemLogits), $topK);
+            [$scores, $indices] = $logits->softmax()->topk($topK);
 
-            $toReturn[] = array_map(function ($score) use ($ids, $maskTokenIndex) {
+            $toReturn = [];
+
+            foreach ($indices as $i => $index) {
                 $sequence = $ids;
-                $sequence[$maskTokenIndex] = $score[0];
+                $sequence[$maskTokenIndex] = $index;
 
-                return [
-                    'score' => $score[1],
-                    'token' => $score[0],
-                    'token_str' => $this->tokenizer->decode([$score[0]], skipSpecialTokens: true),
+                $toReturn[] = [
+                    'score' => $scores[$i],
+                    'token' => $index,
+                    'token_str' => $this->tokenizer->decode([$index], skipSpecialTokens: true),
                     'sequence' => $this->tokenizer->decode($sequence, skipSpecialTokens: true),
                 ];
-            }, $scores);
+            }
         }
 
         return is_array($inputs) ? $toReturn : $toReturn[0];
diff --git a/src/Pipelines/ImageClassificationPipeline.php b/src/Pipelines/ImageClassificationPipeline.php
@@ -8,6 +8,7 @@
 use Codewithkyrian\Transformers\Models\Output\SequenceClassifierOutput;
 use Codewithkyrian\Transformers\Utils\Math;
 use function Codewithkyrian\Transformers\Utils\prepareImages;
+use function Codewithkyrian\Transformers\Utils\timeUsage;
 
 /**
  * Image classification pipeline using any `AutoModelForImageClassification`.
@@ -61,6 +62,7 @@ public function __invoke(array|string $inputs, ...$args): array
 
         ['pixel_values' => $pixelValues] = ($this->processor)($preparedImages);
 
+
         /** @var SequenceClassifierOutput $output */
         $output = $this->model->__invoke(['pixel_values' => $pixelValues]);
 
@@ -69,15 +71,13 @@ public function __invoke(array|string $inputs, ...$args): array
         $toReturn = [];
 
         foreach ($output->logits as $batch) {
-            $scores = Math::getTopItems($batch->softmax()->toArray(), $topK);
-
-            $values = array_map(
-                fn($score) => [
-                    'label' => $id2label[$score[0]],
-                    'score' => $score[1]
-                ],
-                $scores
-            );
+            [$scores, $indices] = $batch->softmax()->topk($topK);
+
+            $values = [];
+
+            foreach ($indices as $i => $index) {
+                $values[] = ['label' => $id2label[$index], 'score' => $scores[$i]];
+            }
 
 
             if ($topK === 1) {
diff --git a/src/Pipelines/TextClassificationPipeline.php b/src/Pipelines/TextClassificationPipeline.php
@@ -70,20 +70,22 @@ public function __invoke(array|string $inputs, ...$args): array
         $problemType = $this->model->config['problem_type'] ?? 'single_label_classification';
 
         $activationFunction = $problemType == 'multi_label_classification' ?
-            fn(Tensor $batch) => $batch->sigmoid()->toArray() :
-            fn(Tensor $batch) => $batch->softmax()->toArray();
+            fn(Tensor $batch) => $batch->sigmoid() :
+            fn(Tensor $batch) => $batch->softmax();
 
         $id2label = $this->model->config['id2label'];
         $toReturn = [];
 
         foreach ($outputs->logits as $batch) {
             $output = $activationFunction($batch);
-            $scores = Math::getTopItems($output, $topK);
 
-            $values = array_map(
-                fn($score) => ['label' => $id2label[$score[0]], 'score' => $score[1],],
-                $scores
-            );
+            [$scores, $indices] = $output->topk($topK);
+
+            $values = [];
+
+            foreach ($indices as $i => $index) {
+                $values[] = ['label' => $id2label[$index], 'score' => $scores[$i]];
+            }
 
             if ($topK === 1) {
                 $toReturn = $values;
diff --git a/src/Processors/Processor.php b/src/Processors/Processor.php
@@ -61,7 +61,7 @@ public static function postProcessObjectDetection(ObjectDetectionOutput $outputs
                 $probs = [];
                 if ($isZeroShot) {
                     // Get indices of classes with high enough probability
-                    $logitSigmoid = Math::sigmoid($logit->toArray());
+                    $logitSigmoid = $logit->sigmoid();
                     foreach ($logitSigmoid as $k => $prob) {
                         if ($prob > $threshold) {
                             $indices[] = $k;
diff --git a/src/Transformers.php b/src/Transformers.php
@@ -8,9 +8,48 @@
 use Codewithkyrian\Transformers\Utils\Image;
 use Codewithkyrian\Transformers\Utils\ImageDriver;
 use OnnxRuntime\Vendor;
+use function Codewithkyrian\Transformers\Utils\joinPaths;
 
 class Transformers
 {
+    public const VERSION = '0.4.0';
+    public const  LIBS_DIR = __DIR__ . '/../libs/';
+
+    protected const LIBRARIES = [
+        'x86_64-darwin' => [
+            'archive' => 'libraries-osx-x86_64-{{version}}.tar.gz',
+            'checksum' => 'f72a2bcca40e2650756c6b96c69ef031236aaab1b98673e744da4eef0c4bddbd',
+            'rindowmatlib.serial' => 'rindow-matlib-Darwin-1.0.0/lib/librindowmatlib_serial.dylib',
+            'rindowmatlib.openmp' => 'rindow-matlib-Darwin-1.0.0/lib/librindowmatlib_openmp.dylib',
+            'openblas.serial' => 'openblas-osx-x86_64-0.3.27/lib/libopenblas_serial.dylib',
+            'openblas.openmp' => 'openblas-osx-x86_64-0.3.27/lib/libopenblas_openmp.dylib',
+            'openblas.include' => 'openblas-osx-x86_64-0.3.27/include/openblas.h',
+            'lapacke.serial' => 'openblas-osx-x86_64-0.3.27/lib/libopenblas_serial.dylib',
+            'lapacke.openmp' => 'openblas-osx-x86_64-0.3.27/lib/libopenblas_openmp.dylib'
+        ],
+        'arm64-darwin' => [
+            'archive' => 'libraries-osx-arm64-{{version}}.tar.gz',
+            'checksum' => 'f72a2bcca40e2650756c6b96c69ef031236aaab1b98673e744da4eef0c4bddbd',
+            'rindowmatlib.serial' => 'rindow-matlib-Darwin-1.0.0/lib/librindowmatlib_serial.dylib',
+            'rindowmatlib.openmp' => 'rindow-matlib-Darwin-1.0.0/lib/librindowmatlib_openmp.dylib',
+            'openblas.serial' => 'openblas-osx-arm64-0.3.27/lib/libopenblas_serial.dylib',
+            'openblas.openmp' => 'openblas-osx-arm64-0.3.27/lib/libopenblas_openmp.dylib',
+            'openblas.include' => 'openblas-osx-arm64-0.3.27/include/openblas.h',
+            'lapacke.serial' => 'openblas-osx-arm64-0.3.27/lib/libopenblas_serial.dylib',
+            'lapacke.openmp' => 'openblas-osx-arm64-0.3.27/lib/libopenblas_openmp.dylib'
+        ],
+        'x86_64-linux' => [
+
+        ],
+        'aarch64-linux' => [
+
+        ],
+        'x64-windows' => [
+
+        ]
+    ];
+
+
     public static string $cacheDir = '.transformers-cache';
 
     public static string $remoteHost = 'https://huggingface.co';
@@ -108,13 +147,16 @@ public function setImageDriver(ImageDriver $imageDriver): static
         return $this;
     }
 
-    public static function platform($key): string
+    public static function getLib(string $key): string
     {
         $platformKey = match (PHP_OS_FAMILY) {
             'Windows' => 'x64-windows',
             'Darwin' => php_uname('m') == 'x86_64' ? 'x86_64-darwin' : 'arm64-darwin',
             default => php_uname('m') == 'x86_64' ? 'x86_64-linux' : 'aarch64-linux',
         };
-        return Vendor::PLATFORMS[$platformKey][$key];
+
+        $filename = self::LIBRARIES[$platformKey][$key];
+
+        return joinPaths(Transformers::LIBS_DIR, $filename);
     }
 }
diff --git a/src/Utils/Tensor.php b/src/Utils/Tensor.php
diff --git a/src/Utils/TensorService.php b/src/Utils/TensorService.php

Original file line number	Diff line number	Diff line change
`@@ -21,12 +21,11 @@ public function sample(Tensor $logits, int $index): array`
`21`	`21`	`{`
`22`	`22`	`// NOTE: no need to do log_softmax here since we only take the maximum`
`23`	`23`	`$logs = $this->getLogits($logits, $index);`
`24`		`- $argmax = array_search(max($logs), $logs);`
`25`	`24`
`26`	`25`	`// Note: score is meaningless in this context, since we are performing`
`27`	`26`	`// greedy search (p = 1 => log(p) = 0)`
`28`	`27`	`return [`
`29`		`- [$argmax, 0]`
	`28`	`+ [$logs->argMax(), 0]`
`30`	`29`	`];`
`31`	`30`	`}`
`32`	`31`	`}`