Updated libraries (km and others)

alkampfergit · alkampfergit · commit ece722ccc1b4 · 2024-12-12T18:12:14.000+01:00
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -4,20 +4,21 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="Alkampfer.KernelMemory.ElasticSearch" Version="0.9.1" />
-    <PackageVersion Include="Azure.AI.OpenAI" Version="2.0.0-beta.5" />
+    <PackageVersion Include="Azure.AI.OpenAI" Version="2.1.0-beta.1" />
     <PackageVersion Include="CommandDotNet.Spectre" Version="3.0.2" />
-    <PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.9.1" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.77.241004.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.77.241004.1" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Abstractions" Version="0.94.241201.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.AI.AzureOpenAI" Version="0.94.241201.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.94.241201.1" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.21.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.21.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.21.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.21.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.21.1" />
+    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.26.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.26.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Yaml" Version="1.26.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.26.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Core" Version="1.26.0" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />
     <PackageVersion Include="TiktokenSharp" Version="1.1.4" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
@@ -26,12 +27,12 @@
   <ItemGroup>
     <PackageVersion Include="coverlet.msbuild" Version="6.0.2" />
     <PackageVersion Include="fasterflect" Version="3.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.abstractions" Version="2.0.3" />
     <PackageVersion Include="Moq" Version="4.20.72" />
-    <PackageVersion Include="Xunit.DependencyInjection" Version="9.4.0" />
+    <PackageVersion Include="Xunit.DependencyInjection" Version="9.7.0" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2" />
   </ItemGroup>
diff --git a/src/KernelMemory.Extensions.ConsoleTest/Helper/TextCleanerHandler.cs b/src/KernelMemory.Extensions.ConsoleTest/Helper/TextCleanerHandler.cs
@@ -23,14 +23,14 @@ public TextCleanerHandler(
 
     public string StepName => _name;
 
-    public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default)
+    public async Task<(ReturnType returnType, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken)
     {
         _log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId);
 
         if (pipeline.Files.Count == 0)
         {
             _log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId);
-            return (true, pipeline);
+            return (ReturnType.Success, pipeline);
         }
 
         foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)
@@ -81,6 +81,6 @@ public TextCleanerHandler(
                 uploadedFile.GeneratedFiles.Add(file.Key, file.Value);
             }
         }
-        return (true, pipeline);
+        return (ReturnType.Success, pipeline);
     }
 }
diff --git a/src/KernelMemory.Extensions.ConsoleTest/KernelMemory.Extensions.ConsoleTest.csproj b/src/KernelMemory.Extensions.ConsoleTest/KernelMemory.Extensions.ConsoleTest.csproj
@@ -12,8 +12,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CommandDotNet.Spectre"  />
+    <PackageReference Include="CommandDotNet.Spectre" />
     <PackageReference Include="Alkampfer.KernelMemory.ElasticSearch" />
+    <PackageReference Include="Microsoft.KernelMemory.AI.AzureOpenAI" />
   </ItemGroup>
 
 </Project>
diff --git a/src/KernelMemory.Extensions.FunctionalTests/Anthropic/AnthropicTests.cs b/src/KernelMemory.Extensions.FunctionalTests/Anthropic/AnthropicTests.cs
@@ -1,6 +1,5 @@
 ﻿using KernelMemory.ElasticSearch.Anthropic;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.KernelMemory.AI.Anthropic;
 
 namespace KernelMemory.Extensions.FunctionalTests;
 
diff --git a/src/KernelMemory.Extensions/Anthropic/AnthropicTextGeneration.cs b/src/KernelMemory.Extensions/Anthropic/AnthropicTextGeneration.cs
@@ -17,7 +17,7 @@ internal class AnthropicTextGeneration : ITextGenerator
     /// We do not have cohere tokenizer directly in C# - in this first version we use gpt4 tokenizer
     /// and we know that this is a raw approximation but we only need to count.
     /// </summary>
-    private static readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
+    private static readonly Tokenizer _tokenizer = Microsoft.ML.Tokenizers.TiktokenTokenizer.CreateForModel("gpt-4");
 
     public AnthropicTextGeneration(
         RawAnthropicHttpClient rawAnthropicHttpClient,
diff --git a/src/KernelMemory.Extensions/Anthropic/ClaudeContextualRetrievalHandler.cs b/src/KernelMemory.Extensions/Anthropic/ClaudeContextualRetrievalHandler.cs
@@ -1,11 +1,9 @@
 using KernelMemory.ElasticSearch.Anthropic;
 using Microsoft.Extensions.Logging;
-using Microsoft.KernelMemory.AI.Anthropic;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -30,16 +28,16 @@ public ClaudeContextualRetrievalHandler(
         _log = log ?? DefaultLogger<ClaudeContextualRetrievalHandler>.Instance; ;
     }
 
-    public string StepName => _name;
-
-    public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default)
+    public string StepName => _name;
+
+    public async Task<(ReturnType returnType, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken)
     {
         _log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId);
 
         if (pipeline.Files.Count == 0)
         {
             _log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId);
-            return (true, pipeline);
+            return (ReturnType.Success, pipeline);
         }
 
         foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)
@@ -86,7 +84,7 @@ public ClaudeContextualRetrievalHandler(
                 }
             }
         }
-        return (true, pipeline);
+        return (ReturnType.Success, pipeline);
     }
 
     private async Task<string> EnrichTextAsync(
diff --git a/src/KernelMemory.Extensions/Helper/MicrosoftMlTiktokenTokenizer.cs b/src/KernelMemory.Extensions/Helper/MicrosoftMlTiktokenTokenizer.cs
@@ -17,7 +17,7 @@ public class MicrosoftMlTiktokenTokenizer : ITextTokenizer
 
         public MicrosoftMlTiktokenTokenizer(string baseModelName)
         {
-            _tikToken = TiktokenTokenizer.CreateForModel(baseModelName);
+            _tikToken = Microsoft.ML.Tokenizers.TiktokenTokenizer.CreateForModel(baseModelName);
         }
 
         public int CountTokens(string text)

Original file line number	Diff line number	Diff line change
`@@ -23,14 +23,14 @@ public TextCleanerHandler(`
`23`	`23`
`24`	`24`	`public string StepName => _name;`
`25`	`25`
`26`		`- public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default)`
	`26`	`+ public async Task<(ReturnType returnType, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken)`
`27`	`27`	`{`
`28`	`28`	`_log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId);`
`29`	`29`
`30`	`30`	`if (pipeline.Files.Count == 0)`
`31`	`31`	`{`
`32`	`32`	`_log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId);`
`33`		`- return (true, pipeline);`
	`33`	`+ return (ReturnType.Success, pipeline);`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)`
`@@ -81,6 +81,6 @@ public TextCleanerHandler(`
`81`	`81`	`uploadedFile.GeneratedFiles.Add(file.Key, file.Value);`
`82`	`82`	`}`
`83`	`83`	`}`
`84`		`- return (true, pipeline);`
	`84`	`+ return (ReturnType.Success, pipeline);`
`85`	`85`	`}`
`86`	`86`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,9 @@`
`1`	`1`	`using KernelMemory.ElasticSearch.Anthropic;`
`2`	`2`	`using Microsoft.Extensions.Logging;`
`3`		`-using Microsoft.KernelMemory.AI.Anthropic;`
`4`	`3`	`using Microsoft.KernelMemory.Diagnostics;`
`5`	`4`	`using Microsoft.KernelMemory.Pipeline;`
`6`	`5`	`using System.Collections.Generic;`
`7`	`6`	`using System.Linq;`
`8`		`-using System.Text;`
`9`	`7`	`using System.Threading;`
`10`	`8`	`using System.Threading.Tasks;`
`11`	`9`
`@@ -30,16 +28,16 @@ public ClaudeContextualRetrievalHandler(`
`30`	`28`	`_log = log ?? DefaultLogger<ClaudeContextualRetrievalHandler>.Instance; ;`
`31`	`29`	`}`
`32`	`30`
`33`		`- public string StepName => _name;`
`34`		`-`
`35`		`- public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default)`
	`31`	`+ public string StepName => _name;`
	`32`	`+`
	`33`	`+ public async Task<(ReturnType returnType, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken)`
`36`	`34`	`{`
`37`	`35`	`_log.LogDebug("Partitioning text, pipeline '{0}/{1}'", pipeline.Index, pipeline.DocumentId);`
`38`	`36`
`39`	`37`	`if (pipeline.Files.Count == 0)`
`40`	`38`	`{`
`41`	`39`	`_log.LogWarning("Pipeline '{0}/{1}': there are no files to process, moving to next pipeline step.", pipeline.Index, pipeline.DocumentId);`
`42`		`- return (true, pipeline);`
	`40`	`+ return (ReturnType.Success, pipeline);`
`43`	`41`	`}`
`44`	`42`
`45`	`43`	`foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)`
`@@ -86,7 +84,7 @@ public ClaudeContextualRetrievalHandler(`
`86`	`84`	`}`
`87`	`85`	`}`
`88`	`86`	`}`
`89`		`- return (true, pipeline);`
	`87`	`+ return (ReturnType.Success, pipeline);`
`90`	`88`	`}`
`91`	`89`
`92`	`90`	`private async Task<string> EnrichTextAsync(`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ public class MicrosoftMlTiktokenTokenizer : ITextTokenizer`
`17`	`17`
`18`	`18`	`public MicrosoftMlTiktokenTokenizer(string baseModelName)`
`19`	`19`	`{`
`20`		`- _tikToken = TiktokenTokenizer.CreateForModel(baseModelName);`
	`20`	`+ _tikToken = Microsoft.ML.Tokenizers.TiktokenTokenizer.CreateForModel(baseModelName);`
`21`	`21`	`}`
`22`	`22`
`23`	`23`	`public int CountTokens(string text)`