forked from CodeWithKyrian/transformers-php
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathTemplateProcessing.php
More file actions
66 lines (53 loc) · 2.14 KB
/
TemplateProcessing.php
File metadata and controls
66 lines (53 loc) · 2.14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
<?php
declare(strict_types=1);
namespace Codewithkyrian\Transformers\PostProcessors;
/**
* Post processor that replaces special tokens in a template with actual tokens.
*/
class TemplateProcessing extends PostProcessor
{
/**
* @var array The template for a single sequence of tokens.
*/
public array $single;
/**
* @var array The template for a pair of sequences of tokens.
*/
protected array $pair;
public function __construct(array $config)
{
parent::__construct($config);
$this->single = $config['single'];
$this->pair = $config['pair'];
}
/**
* Replaces special tokens in the template with actual tokens.
* @param string[] $tokens The input tokens.
* @param string[]|null $tokenPair The input tokens for the second sequence in a pair.
* @param bool $addSpecialTokens Whether to add the special tokens associated with the corresponding model.
* @return PostProcessedOutput
*/
public function postProcess(array $tokens, ?array $tokenPair = null, bool $addSpecialTokens = true): PostProcessedOutput
{
$type = $tokenPair === null ? $this->single : $this->pair;
$processedTokens = [];
$types = [];
foreach ($type as $item) {
if (isset($item['SpecialToken'])) {
if ($addSpecialTokens) {
$processedTokens[] = $item['SpecialToken']['id'];
$types[] = $item['SpecialToken']['type_id'];
}
} elseif (isset($item['Sequence'])) {
if ($item['Sequence']['id'] === 'A') {
$processedTokens = array_merge($processedTokens, $tokens);
$types = array_merge($types, array_fill(0, count($tokens), $item['Sequence']['type_id']));
} elseif ($item['Sequence']['id'] === 'B') {
$processedTokens = array_merge($processedTokens, $tokenPair);
$types = array_merge($types, array_fill(0, count($tokenPair), $item['Sequence']['type_id']));
}
}
}
return new PostProcessedOutput($processedTokens, $types);
}
}