Some fixes

Author: alkampfergit

src/KernelMemory.Extensions.ConsoleTest/Samples/CustomSearchPipelineBase.cs

@@ -453,18 +453,18 @@ private static IKernelBuilder CreateBasicKernelBuilder()
             .AddLogger(s => _loggingProvider.CreateHttpRequestBodyLogger(s.GetRequiredService<ILogger<DumpLoggingProvider>>())));
 
         kernelBuilder.Services.AddAzureOpenAIChatCompletion(
-            "GPT35_2",
+            "gpt-4o-mini",
             Dotenv.Get("OPENAI_API_BASE") ?? throw new ConfigurationException("OPENAI_API_BASE missing from .env file"),
             Dotenv.Get("OPENAI_API_KEY") ?? throw new ConfigurationException("OPENAI_API_KEY missing from .env file"),
-            serviceId: "gpt35",
-            modelId: "gpt35");
+            serviceId: "gpt-4o-mini",
+            modelId: "gpt-4o-mini");
 
         kernelBuilder.Services.AddAzureOpenAIChatCompletion(
-            "GPT4o", //"GPT35_2",//"GPT42",
+            "gpt-4o", //"GPT35_2",//"GPT42",
             Dotenv.Get("OPENAI_API_BASE") ?? throw new ConfigurationException("OPENAI_API_BASE missing from .env file"),
             Dotenv.Get("OPENAI_API_KEY") ?? throw new ConfigurationException("OPENAI_API_KEY missing from .env file"),
             serviceId: "default",
-            modelId: "gpt4o");
+            modelId: "gpt-4o");
 
         return kernelBuilder;
     }

src/KernelMemory.Extensions.FunctionalTests/Cohere/CohereTests.cs

@@ -139,7 +139,7 @@ public async Task Basic_cohere_embed_test()
     public void Tokenizer_raw_test()
     {
         CohereTokenizer tokenizer = new(_httpClientFactory);
-        var count = tokenizer.CountToken("command-r-plus", "Now I'm using CommandR+ tokenizer");
+        var count = tokenizer.CountToken(CohereModels.CommandDefault, "Now I'm using CommandR+ tokenizer");
         Assert.Equal(8, count);
     }
 

src/KernelMemory.Extensions/Cohere/CohereTokenizer.cs

@@ -16,23 +16,28 @@ public class CohereTokenizer
 {
     public Dictionary<string, TiktokenTokenizer> Tokenizers { get; set; } = new();
 
-    public CohereTokenizer(IHttpClientFactory httpClientFactory)
-    {
-        var tokenizerFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "command-r-plus.tiktoken");
-        var tokenizerExtraFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "command-r-plus.tiktoken.extra");
-
-        DownloadCohereTokenizerSpecifcationFileAndConvertToTiktoken(
-            httpClientFactory,
-            "https://storage.googleapis.com/cohere-public/tokenizers/command-r-plus.json",
-            tokenizerFile,
-            tokenizerExtraFile);
-
-        //now we need to load the tokenizer, first of all we load the extra data
-        var extraData = File.ReadAllText(tokenizerExtraFile);
-        var ed = JsonSerializer.Deserialize<ExtraTokenizerData>(extraData)!;
-
-        var tiktoken = TiktokenTokenizer.Create(tokenizerFile, null, null, specialTokens: ed.GetSpecialToken());
-        Tokenizers["command-r-plus"] = tiktoken;
+    public CohereTokenizer(IHttpClientFactory httpClientFactory, IEnumerable<string>? modelNames = null)
+    {
+        var models = modelNames?.ToArray() ?? new[] { CohereModels.CommandDefault };
+
+        foreach (var modelName in models)
+        {
+            var tokenizerFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"{modelName}.tiktoken");
+            var tokenizerExtraFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"{modelName}.tiktoken.extra");
+
+            DownloadCohereTokenizerSpecifcationFileAndConvertToTiktoken(
+                httpClientFactory,
+                $"https://storage.googleapis.com/cohere-public/tokenizers/{modelName}.json",
+                tokenizerFile,
+                tokenizerExtraFile);
+
+            //now we need to load the tokenizer, first of all we load the extra data
+            var extraData = File.ReadAllText(tokenizerExtraFile);
+            var ed = JsonSerializer.Deserialize<ExtraTokenizerData>(extraData)!;
+
+            var tiktoken = TiktokenTokenizer.Create(tokenizerFile, null, null, specialTokens: ed.GetSpecialToken());
+            Tokenizers[modelName] = tiktoken;
+        }
     }
 
     private static void DownloadCohereTokenizerSpecifcationFileAndConvertToTiktoken(
@@ -41,7 +46,7 @@ private static void DownloadCohereTokenizerSpecifcationFileAndConvertToTiktoken(
         string tokenizerFile,
         string tokenizerExtraFile)
     {
-        if (!File.Exists(tokenizerFile))
+        if (!File.Exists(tokenizerFile))
         {
             var client = httpClientFactory.CreateClient();
             var response = client.GetAsync(definitionLocation).Result;

src/KernelMemory.Extensions/Cohere/RawCohereClientDtos.cs

@@ -95,7 +95,7 @@ internal string Describe()
     public string Message { get; set; }
 
     [JsonPropertyName("model")]
-    public string Model { get; set; } = "command-r-plus";
+    public string Model { get; set; } = CohereModels.CommandDefault;
 
     [JsonPropertyName("documents")]
     public List<RagDocument> Documents { get; set; }
@@ -425,6 +425,11 @@ public static class CohereModels
     public const string EmbedEnglishV2 = "embed-english-v2.0";
     public const string EmbedEnglishLightV2 = "embed-english-light-v2.0";
     public const string EmbedMultilingualV2 = "embed-multilingual-v2.0";
+    // Command/chat family models
+
+    public const string CommandA032025 = "command-a-03-2025";
+    // Default command model used by the RAG/Chat client
+    public const string CommandDefault = CommandA032025;
 }
 
 public class CohereEmbedRequest