Seperate AutoModel class for CTC and bug fixes for WhisperTokenizer

Author: CodeWithKyrian

examples/pipelines/asr.php

@@ -11,7 +11,9 @@
 
 ini_set('memory_limit', '-1');
 
-$transcriber = pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
+//$transcriber = pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
+$transcriber = pipeline('automatic-speech-recognition', 'Xenova/whisper-base');
+//$transcriber = pipeline('automatic-speech-recognition', 'Xenova/wav2vec2-large-xlsr-53-english');
 
 $audioUrl = __DIR__ . '/../sounds/kyrian-dev.wav';
 //$audioUrl = __DIR__ . '/../sounds/jfk.wav';
@@ -24,6 +26,6 @@
 //$audioUrl = __DIR__ . '/../sounds/dataset1.wav';
 
 $streamer = StdOutStreamer::make();
-$output = $transcriber($audioUrl, maxNewTokens: 256, chunkLengthSecs: 18,  returnTimestamps: 'word');
+$output = $transcriber($audioUrl, maxNewTokens: 256, chunkLengthSecs: 20, returnTimestamps: 'word');
 
 dd($output, timeUsage(), memoryUsage());

src/FeatureExtractors/Wav2Vec2FeatureExtractor.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\FeatureExtractors;
+
+use Codewithkyrian\Transformers\Tensor\Tensor;
+use function Codewithkyrian\Transformers\Utils\timeUsage;
+
+class Wav2Vec2FeatureExtractor extends FeatureExtractor
+{
+    /**
+     *  Extracts features from a given audio using the provided configuration.
+     * @param Tensor $waveform The audio tensor to extract features from.
+     * @return Tensor[] The extracted features.
+     */
+    public function __invoke(Tensor $waveform): array
+    {
+        // zero-mean and unit-variance normalization
+        if ($this->config['do_normalize'])
+        {
+            $mean = $waveform->mean();
+
+            //calculate the variance
+//            $variance = $waveform->add(-$mean)->pow(2)->mean();
+            $variance = 0;
+            for ($i = 0; $i < $waveform->size(); $i++) {
+                $variance += pow($waveform[$i] - $mean, 2);
+            }
+            $variance /= $waveform->size();
+
+            //normalize the waveform
+            $waveform = $waveform->add(-$mean)->multiply(1.0 / sqrt($variance + 1e-7));
+        }
+
+        $shape = [1, $waveform->size()];
+
+        return [
+            'input_values' => $waveform->reshape($shape),
+            'attention_mask' => Tensor::ones($shape, dtype: Tensor::int64)
+        ];
+    }
+}

src/Models/Auto/AutoModel.php

@@ -21,6 +21,7 @@ class AutoModel extends PretrainedMixin
         "siglip" => \Codewithkyrian\Transformers\Models\Pretrained\SigLipModel::class,
 
         "audio-spectrogram-transformer" => \Codewithkyrian\Transformers\Models\Pretrained\ASTModel::class,
+        "wav2vec2" => \Codewithkyrian\Transformers\Models\Pretrained\Wav2Vec2Model::class,
 
         'detr' => \Codewithkyrian\Transformers\Models\Pretrained\DETRModel::class,
         'yolos' => \Codewithkyrian\Transformers\Models\Pretrained\YOLOSModel::class,

src/Models/Auto/AutoModelForAudioClassification.php

@@ -9,6 +9,7 @@ class AutoModelForAudioClassification  extends PretrainedMixin
 {
     const MODEL_CLASS_MAPPING = [
         'audio-spectrogram-transformer' => \Codewithkyrian\Transformers\Models\Pretrained\ASTForAudioClassification::class,
+        'wav2vec2' => \Codewithkyrian\Transformers\Models\Pretrained\Wav2Vec2ForSequenceClassification::class,
     ];
 
     const MODEL_CLASS_MAPPINGS = [

src/Models/Auto/AutoModelForCTC.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Auto;
+
+class AutoModelForCTC extends PretrainedMixin
+{
+    const MODEL_CLASS_MAPPING = [
+        'wav2vec2' => \Codewithkyrian\Transformers\Models\Pretrained\Wav2Vec2ForCTC::class,
+    ];
+
+    const MODEL_CLASS_MAPPINGS = [
+        self::MODEL_CLASS_MAPPING,
+    ];
+}

src/Models/Auto/PretrainedMixin.php

@@ -104,6 +104,7 @@ protected static function getModelArchitecture($modelClass): ModelArchitecture
             in_array($modelClass, AutoModelForImageClassification::MODEL_CLASS_MAPPING) => ModelArchitecture::EncoderOnly,
             in_array($modelClass, AutoModelForAudioClassification::MODEL_CLASS_MAPPING) => ModelArchitecture::EncoderOnly,
             in_array($modelClass, AutoModelForSpeechSeq2Seq::MODEL_CLASS_MAPPING) => ModelArchitecture::Seq2SeqLM,
+            in_array($modelClass, AutoModelForCTC::MODEL_CLASS_MAPPING) => ModelArchitecture::EncoderOnly,
 
             default => ModelArchitecture::EncoderOnly,
         };

src/Models/Output/CasualLMOutput.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Output;
+
+use Codewithkyrian\Transformers\Tensor\Tensor;
+
+class CasualLMOutput implements ModelOutput
+{
+    public function __construct(public readonly Tensor $logits)
+    {
+    }
+
+    public static function fromOutput(array $array): self
+    {
+        return new self($array['logits']);
+    }
+}

src/Models/Pretrained/Wav2Vec2ForAudioFrameClassification.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+use Codewithkyrian\Transformers\Models\Output\ModelOutput;
+use Codewithkyrian\Transformers\Models\Output\TokenClassifierOutput;
+
+/**
+ * Wav2Vec2 Model with a frame classification head on top for tasks like Speaker Diarization.
+ */
+class Wav2Vec2ForAudioFrameClassification extends Wav2Vec2PretrainedModel
+{
+public function __invoke(array $modelInputs): array|ModelOutput
+{
+    return TokenClassifierOutput::fromOutput(parent::__invoke($modelInputs));
+}
+}

src/Models/Pretrained/Wav2Vec2ForCTC.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+use Codewithkyrian\Transformers\Models\Output\CasualLMOutput;
+use Codewithkyrian\Transformers\Models\Output\ModelOutput;
+
+class Wav2Vec2ForCTC extends Wav2Vec2PretrainedModel
+{
+//    public function __invoke(array $modelInputs): array|ModelOutput
+//    {
+//        return CasualLMOutput::fromOutput(parent::__invoke($modelInputs));
+//    }
+}

src/Models/Pretrained/Wav2Vec2ForSequenceClassification.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+
+namespace Codewithkyrian\Transformers\Models\Pretrained;
+
+use Codewithkyrian\Transformers\Models\Output\ModelOutput;
+use Codewithkyrian\Transformers\Models\Output\SequenceClassifierOutput;
+
+class Wav2Vec2ForSequenceClassification extends Wav2Vec2PretrainedModel
+{
+    public function __invoke(array $modelInputs): array|ModelOutput
+    {
+        return SequenceClassifierOutput::fromOutput(parent::__invoke($modelInputs));
+    }
+}