Skip to content

Commit 91f82f1

Browse files
CodeWithKyrianCodeWithKyrian
committed
Allow normal PHP buffer to work with inference session
1 parent 32d8cfd commit 91f82f1

5 files changed

Lines changed: 55 additions & 32 deletions

File tree

‎src/Generation/Samplers/Sampler.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public function getLogits(Tensor $logits, int $index): Tensor
6666
$logs = $logits->newSlice($start, $size);
6767

6868
if ($this->generationConfig->temperature > 0) {
69-
$logs = $logs->divide($this->generationConfig->temperature);
69+
$logs = $logs->multiply(1 / $this->generationConfig->temperature);
7070
}
7171

7272
// Remove all dimensions of 1, leaving a flat 1D array of vocab_size

‎src/Utils/InferenceSession.php‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -391,23 +391,35 @@ private function convertInputTensorToOnnxTensor($inputFeed, &$refs)
391391
$this->checkStatus(($this->api->CreateTensorAsOrtValue)($this->allocator, $inputNodeShape, $ndim, $typeEnum, \FFI::addr($inputTensor[$idx])));
392392
$this->checkStatus(($this->api->FillStringTensor)($inputTensor[$idx], $inputTensorValues, count($inputTensorValues)));
393393

394-
$refs[] = $inputTensorValues;
395394
} else {
396395

397396
$inputTypes = array_flip(array_map(fn($v) => "tensor($v)", $this->elementDataTypes()));
398397

399398
if (isset($inputTypes[$inp['type']])) {
400399
$typeEnum = $inputTypes[$inp['type']];
400+
$castType = $this->castTypes()[$typeEnum];
401401
$phpTensorType = self::ONNX_TENSOR_TYPE_TO_PHP_TENSOR_MAP[$typeEnum];
402402
$input = $input->to($phpTensorType);
403403
} else {
404404
$this->unsupportedType('input', $inp['type']);
405405
}
406406

407-
$this->checkStatus(($this->api->CreateTensorWithDataAsOrtValue)($allocatorInfo, $input->buffer()->data(), $input->buffer()->byteSize(), $inputNodeShape, $ndim, $typeEnum, \FFI::addr($inputTensor[$idx])));
407+
408+
if ($size === 0) {
409+
$inputTensorValues = $this->ffi->new("void *");
410+
} else {
411+
$inputTensorValues = $this->ffi->new("{$castType}[$size]");
412+
}
413+
414+
$inputDump = $input->buffer()->dump();
415+
416+
\FFI::memcpy($inputTensorValues, $inputDump, strlen($inputDump));
417+
418+
$this->checkStatus(($this->api->CreateTensorWithDataAsOrtValue)($allocatorInfo, $inputTensorValues, \FFI::sizeof($inputTensorValues), $inputNodeShape, $ndim, $typeEnum, \FFI::addr($inputTensor[$idx])));
408419
}
409420

410421
$refs[] = $inputNodeShape;
422+
$refs[] = $inputTensorValues;
411423

412424
$idx++;
413425
}
@@ -484,7 +496,7 @@ private function createFromOnnxValue($outPtr)
484496

485497
$buffer = Tensor::newBuffer($outputTensorSize, $phpTensorType);
486498

487-
$stringPtr = \FFI::string($arr, $buffer->byteSize());
499+
$stringPtr = \FFI::string($arr, \FFI::sizeof($arr));
488500

489501
$buffer->load($stringPtr);
490502

‎src/Utils/Tensor.php‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,27 @@ public function toBufferArray()
387387
throw new Exception('toBufferArray is not implemented yet');
388388
}
389389

390+
public static function fill(array $shape, float|int $value, ?int $dtype = null): static
391+
{
392+
$mo = self::mo();
393+
394+
$ndArray = $mo->full($shape, $value, $dtype);
395+
396+
return new static($ndArray->buffer(), $ndArray->dtype(), $ndArray->shape(), $ndArray->offset());
397+
}
398+
399+
public static function repeat(Tensor|array $tensor, int $repeats, ?int $axis = null): static
400+
{
401+
$mo = self::mo();
402+
403+
if (is_array($tensor)) {
404+
$tensor = $mo->array($tensor);
405+
}
406+
407+
$ndArray = $mo->la()->repeat($tensor, $repeats, $axis);
408+
409+
return new static($ndArray->buffer(), $ndArray->dtype(), $ndArray->shape(), $ndArray->offset());
410+
}
390411

391412
/**
392413
* Return a one matrix with the given shape.
@@ -555,7 +576,11 @@ public function add(Tensor|float|int $other): static
555576
{
556577
$mo = self::mo();
557578

558-
$ndArray = is_scalar($other) ? $mo->op($this, '+', $other) : $mo->add($this, $other);
579+
if ($other instanceof Tensor) {
580+
$ndArray = $mo->la()->add($this, $other);
581+
} else {
582+
$ndArray = $mo->la()->increment($this, $other);
583+
}
559584

560585
return new static($ndArray->buffer(), $ndArray->dtype(), $ndArray->shape(), $ndArray->offset());
561586
}
@@ -581,20 +606,15 @@ public function sigmoid(): self
581606
*
582607
* @return self
583608
*/
584-
public function multiply(float|int $scalar): self
609+
public function multiply(Tensor|float|int $scalar): self
585610
{
586611
$mo = self::mo();
587612

588-
$ndArray = $mo->la()->scal($scalar, $this);
589-
590-
return new static($ndArray->buffer(), $ndArray->dtype(), $ndArray->shape(), $ndArray->offset());
591-
}
592-
593-
public function divide(float|int $scalar): self
594-
{
595-
$mo = self::mo();
596-
597-
$ndArray = $mo->la()->scal(1 / $scalar, $this);
613+
if ($scalar instanceof Tensor) {
614+
$ndArray = $mo->la()->multiply($this, $scalar);
615+
} else {
616+
$ndArray = $mo->la()->scal($scalar, $this);
617+
}
598618

599619
return new static($ndArray->buffer(), $ndArray->dtype(), $ndArray->shape(), $ndArray->offset());
600620
}
@@ -854,7 +874,7 @@ public function meanPooling(Tensor $other): Tensor
854874
return new Tensor($returnedData, $this->dtype(), [$batchSize, $embedAxis]);
855875
}
856876

857-
public function newSlice(array $start, array $size) : Tensor
877+
public function newSlice(array $start, array $size): Tensor
858878
{
859879
$mo = self::mo();
860880

@@ -972,7 +992,7 @@ protected function softmax2D(): static
972992
/**
973993
* @return Tensor[]
974994
*/
975-
public function topk(int $k = null, bool $sorted = true) : array
995+
public function topk(int $k = null, bool $sorted = true): array
976996
{
977997
if ($k === null) {
978998
$k = $this->shape[0];
@@ -1052,7 +1072,7 @@ public function topk(int $k = null, bool $sorted = true) : array
10521072
// Extract top K values and indices from the heap
10531073
for ($j = 0; $j < $k; $j++) {
10541074
$topValues->buffer[$this->offset + ($i * $k) + $j] = $heap[$j]['value'];
1055-
$topIndices->buffer[$this->offset + ($i * $k )+ $j] = $heap[$j]['index'];
1075+
$topIndices->buffer[$this->offset + ($i * $k) + $j] = $heap[$j]['index'];
10561076
}
10571077
}
10581078

‎src/Utils/TensorBuffer.php‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -124,16 +124,6 @@ public function valueSize(): int
124124
return self::$valueSize[$this->dtype];
125125
}
126126

127-
public function byteSize(): int
128-
{
129-
return $this->valueSize() * $this->size;
130-
}
131-
132-
public function data()
133-
{
134-
return $this->data;
135-
}
136-
137127
public function addr(int $offset): FFI\CData
138128
{
139129
return FFI::addr($this->data[$offset]);
@@ -190,14 +180,14 @@ public function offsetSet(mixed $offset, mixed $value): void
190180

191181
public function offsetUnset(mixed $offset): void
192182
{
193-
//$this->assertOffsetIsInt('offsetUnset',$offset);
194183
throw new LogicException("Illegal Operation");
195184
}
196185

197186
public function dump(): string
198187
{
199188
$byte = self::$valueSize[$this->dtype] * $this->size;
200-
$buf = self::$ffi->new("char[$byte]");
189+
if ($byte === 0) return '';
190+
$buf = FFI::new("char[$byte]");
201191
FFI::memcpy($buf, $this->data, $byte);
202192
return FFI::string($buf, $byte);
203193
}

‎src/Utils/TensorService.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ public function __construct(
3434
libFiles: [Transformers::getLib('rindowmatlib.openmp')]
3535
);
3636

37-
$bufferFactory = new TensorBufferFactory();
3837

3938
if (!$openblasFactory->isAvailable()
4039
|| !$mathFactory->isAvailable()
@@ -50,6 +49,8 @@ public function __construct(
5049
);
5150
}
5251

52+
$bufferFactory = new TensorBufferFactory();
53+
5354
parent::__construct(
5455
bufferFactory: $bufferFactory,
5556
openblasFactory: $openblasFactory,

0 commit comments

Comments
 (0)