Create buffer directly when convert model output to tensor

Author: CodeWithKyrian

examples/pipelines/text-generation.php

@@ -13,7 +13,7 @@
 //
 //$generator = pipeline('text-generation', 'Xenova/gpt2');
 $generator = pipeline('text-generation', 'Xenova/Qwen1.5-0.5B-Chat');
-
+//
 $streamer = StdOutStreamer::make();
 
 $messages = [
@@ -24,7 +24,7 @@
 $input = $generator->tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false);
 
 $output = $generator($input,
-//    streamer: $streamer,
+    streamer: $streamer,
     maxNewTokens: 128,
     doSample: true,
     returnFullText: false,

src/Models/Pretrained/PretrainedModel.php

@@ -33,7 +33,6 @@
 use Exception;
 use Symfony\Component\Console\Output\OutputInterface;
 use function Codewithkyrian\Transformers\Utils\array_some;
-use function Codewithkyrian\Transformers\Utils\timeUsage;
 
 /**
  * A base class for pre-trained models that provides the model configuration and an ONNX session.
@@ -282,10 +281,7 @@ public function runSession(InferenceSession $session, array $inputs): array
 
             $outputNames = array_column($session->outputs(), 'name');
 
-            timeUsage();
-            $out = $session->run($outputNames, $inputs);
-            dump(timeUsage(true));
-            return $out;
+            return $session->run($outputNames, $inputs);
         } catch (MissingModelInputException $e) {
             throw $e;
         } catch (Exception $e) {

src/Utils/InferenceSession.php

@@ -130,6 +130,7 @@ public function __destruct()
 
     public function run($outputNames, $inputFeed, $logSeverityLevel = null, $logVerbosityLevel = null, $logid = null, $terminate = null): array
     {
+
         // pointer references
         $refs = [];
 
@@ -167,7 +168,6 @@ public function run($outputNames, $inputFeed, $logSeverityLevel = null, $logVerb
         $output = [];
 
         foreach ($outputTensor as $i => $t) {
-//            $output[] = $this->createFromOnnxValue($t);
             $output[$outputNames[$i]] = $this->createFromOnnxValue($t);
         }
 
@@ -514,13 +514,13 @@ private function fillOutput($ptr, $shape): ?Tensor
             return null;
         }
 
-        $data = [];
+        $buffer = Tensor::newBuffer($bufferSize);
 
         for ($j = 0; $j < $bufferSize; $j++) {
-            $data[] = $ptr[$j];
+            $buffer[$j] = $ptr[$j];
         }
 
-        return new Tensor($data, shape: $shape);
+        return new Tensor($buffer, shape: $shape, offset: 0);
     }
 
     private function createStringsFromOnnxValue($outPtr, $outputTensorSize)

src/Utils/Tensor.php

@@ -42,15 +42,15 @@ public function __construct($array = null, int $dtype = null, array $shape = nul
 
         if (is_array($array) || $array instanceof ArrayObject) {
             $size = $this->countRecursive($array);
-            $this->buffer = $this->newBuffer($size, $dtype);
+            $this->buffer = self::newBuffer($size, $dtype);
             $this->flattenArray($array, $this->buffer);
             $this->offset = 0;
             $shape ??= $this->generateShape($array);
         } elseif (is_numeric($array) || is_bool($array)) {
             if (is_bool($array) && $dtype != NDArray::bool) {
                 throw new InvalidArgumentException("Unmatched dtype with bool value");
             }
-            $this->buffer = $this->newBuffer(1, $dtype);
+            $this->buffer = self::newBuffer(1, $dtype);
             $this->buffer[0] = $array;
             $this->offset = 0;
             $shape = $shape ?? [];
@@ -61,7 +61,7 @@ public function __construct($array = null, int $dtype = null, array $shape = nul
         } elseif ($array === null && $shape !== null) {
             $this->assertShape($shape);
             $size = (int)array_product($shape);
-            $this->buffer = $this->newBuffer($size, $dtype);
+            $this->buffer = self::newBuffer($size, $dtype);
             $this->offset = 0;
         } elseif ($this->isBuffer($array)) {
             if (!is_int($offset))
@@ -107,7 +107,7 @@ function countRecursive($array): int
      * @param int $dtype The data type of the buffer.
      * @return SplFixedArray|OpenBlasBuffer
      */
-    protected function newBuffer(int $size, int $dtype): SplFixedArray|OpenBlasBuffer
+    public static function newBuffer(int $size, ?int $dtype = null): SplFixedArray|OpenBlasBuffer
     {
         if (extension_loaded('rindow_openblas')) {
             return new OpenBlasBuffer($size, $dtype);
@@ -389,7 +389,7 @@ public static function cat(array $tensors, int $axis = 0): Tensor
         // Create a new array to store the accumulated values
         $resultSize = array_product($resultShape);
 
-        $result = $tensors[0]->newBuffer($resultSize, $resultType);
+        $result = self::newBuffer($resultSize, $resultType);
 
         // Create output tensor of same type as first
 
@@ -921,7 +921,7 @@ public function slice(...$slices): Tensor
 
         $newBufferSize = array_reduce($newShape, fn($a, $b) => $a * $b, 1);
 
-        $buffer = $this->newBuffer($newBufferSize, $this->dtype());
+        $buffer = self::newBuffer($newBufferSize, $this->dtype());
         $stride = $this->stride();
 
         for ($i = 0; $i < $newBufferSize; ++$i) {