Skip to content

Commit 7f9e3b8

Browse files
KSemenenkoKSemenenko
committed
local models
1 parent bfcb511 commit 7f9e3b8

21 files changed

Lines changed: 499 additions & 7 deletions

‎Directory.Packages.props‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@
1111
<ItemGroup>
1212
<PackageVersion Include="coverlet.collector" Version="8.0.0" />
1313
<PackageVersion Include="FluentAssertions" Version="8.9.0" />
14+
<PackageVersion Include="LLamaSharp" Version="0.26.0" />
15+
<PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.26.0" />
1416
<PackageVersion Include="ManagedCode.ClaudeCodeSharpSDK.Extensions.AgentFramework" Version="1.1.0-rc4" />
1517
<PackageVersion Include="ManagedCode.ClaudeCodeSharpSDK.Extensions.AI" Version="1.1.0" />
1618
<PackageVersion Include="ManagedCode.CodexSharpSDK.Extensions.AgentFramework" Version="1.1.0-rc4" />
1719
<PackageVersion Include="ManagedCode.CodexSharpSDK.Extensions.AI" Version="1.1.0" />
1820
<PackageVersion Include="ManagedCode.GeminiSharpSDK.Extensions.AgentFramework" Version="0.0.1-rc4" />
1921
<PackageVersion Include="ManagedCode.GeminiSharpSDK.Extensions.AI" Version="0.0.1" />
22+
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.12.2" />
2023
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.3.0" />
2124
<PackageVersion Include="Microsoft.Agents.AI" Version="1.0.0-rc4" />
2225
<PackageVersion Include="Microsoft.Agents.AI.GitHub.Copilot" Version="1.0.0-preview.260311.1" />

‎DotPilot.Core/AgentBuilder/Services/AgentPromptDraftGenerator.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ private static IEnumerable<AgentProviderKind> ResolveProviderPreferences(string
135135
yield return AgentProviderKind.GitHubCopilot;
136136
yield return AgentProviderKind.Gemini;
137137
yield return AgentProviderKind.ClaudeCode;
138+
yield return AgentProviderKind.Onnx;
139+
yield return AgentProviderKind.LlamaSharp;
138140
yield return AgentProviderKind.Debug;
139141
yield break;
140142
}
@@ -145,6 +147,8 @@ private static IEnumerable<AgentProviderKind> ResolveProviderPreferences(string
145147
yield return AgentProviderKind.Gemini;
146148
yield return AgentProviderKind.Codex;
147149
yield return AgentProviderKind.GitHubCopilot;
150+
yield return AgentProviderKind.Onnx;
151+
yield return AgentProviderKind.LlamaSharp;
148152
yield return AgentProviderKind.Debug;
149153
yield break;
150154
}
@@ -153,6 +157,8 @@ private static IEnumerable<AgentProviderKind> ResolveProviderPreferences(string
153157
yield return AgentProviderKind.Gemini;
154158
yield return AgentProviderKind.ClaudeCode;
155159
yield return AgentProviderKind.GitHubCopilot;
160+
yield return AgentProviderKind.Onnx;
161+
yield return AgentProviderKind.LlamaSharp;
156162
yield return AgentProviderKind.Debug;
157163
}
158164

‎DotPilot.Core/ChatSessions/Diagnostics/AgentExecutionLoggingMiddleware.ChatClient.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ private IChatClient WrapChatClient(IChatClient chatClient, AgentRunLogContext ru
4141

4242
private static bool ShouldBridgeSystemInstructions(AgentProviderKind providerKind)
4343
{
44-
return providerKind is AgentProviderKind.Codex or AgentProviderKind.ClaudeCode or AgentProviderKind.Gemini;
44+
return providerKind is AgentProviderKind.Codex or AgentProviderKind.ClaudeCode or AgentProviderKind.Gemini or AgentProviderKind.Onnx or AgentProviderKind.LlamaSharp;
4545
}
4646

4747
private async Task<ChatResponse> LogChatResponseAsync(

‎DotPilot.Core/ChatSessions/Execution/AgentRuntimeConversationFactory.cs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Microsoft.Extensions.AI;
1313
using Microsoft.Extensions.DependencyInjection;
1414
using Microsoft.Extensions.Logging;
15+
using Microsoft.ML.OnnxRuntimeGenAI;
1516
using ClaudeThreadOptions = ManagedCode.ClaudeCodeSharpSDK.Client.ThreadOptions;
1617
using CodexThreadOptions = ManagedCode.CodexSharpSDK.Client.ThreadOptions;
1718
using GeminiApprovalMode = ManagedCode.GeminiSharpSDK.Client.ApprovalMode;
@@ -235,6 +236,21 @@ private IChatClient CreateChatClient(
235236
});
236237
}
237238

239+
if (providerKind == AgentProviderKind.Onnx)
240+
{
241+
var modelPath = ResolveLocalModelPath(providerKind);
242+
return new OnnxRuntimeGenAIChatClient(modelPath);
243+
}
244+
245+
if (providerKind == AgentProviderKind.LlamaSharp)
246+
{
247+
var modelPath = ResolveLocalModelPath(providerKind);
248+
var loggerFactory = serviceProvider.GetService<ILoggerFactory>();
249+
return new LlamaLocalChatClient(
250+
modelPath,
251+
loggerFactory?.CreateLogger<LlamaLocalChatClient>());
252+
}
253+
238254
throw new InvalidOperationException(
239255
string.Format(
240256
CultureInfo.InvariantCulture,
@@ -317,7 +333,23 @@ private string ResolvePlaygroundDirectory(SessionId sessionId)
317333

318334
private static bool ShouldUseFolderChatHistory(AgentProviderKind providerKind)
319335
{
320-
return providerKind == AgentProviderKind.Debug;
336+
return providerKind is AgentProviderKind.Debug or AgentProviderKind.Onnx or AgentProviderKind.LlamaSharp;
337+
}
338+
339+
private static string ResolveLocalModelPath(AgentProviderKind providerKind)
340+
{
341+
var configuration = LocalModelProviderConfigurationReader.Read(providerKind);
342+
if (configuration.IsReady && !string.IsNullOrWhiteSpace(configuration.ModelPath))
343+
{
344+
return configuration.ModelPath;
345+
}
346+
347+
throw new InvalidOperationException(
348+
string.Format(
349+
CultureInfo.InvariantCulture,
350+
"{0} is not configured. Set {1} before starting a local session.",
351+
providerKind.GetDisplayName(),
352+
configuration.PrimaryEnvironmentVariableName));
321353
}
322354

323355
private static string? ResolveExecutablePath(AgentProviderKind providerKind)

‎DotPilot.Core/ChatSessions/Execution/AgentSessionService.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,6 +867,12 @@ private static async ValueTask<AgentProviderKind> ResolveSeedProviderKindAsync(
867867
continue;
868868
}
869869

870+
if (providerKind.IsLocalModelProvider() &&
871+
LocalModelProviderConfigurationReader.Read(providerKind).IsReady)
872+
{
873+
return providerKind;
874+
}
875+
870876
if (!string.IsNullOrWhiteSpace(ResolveExecutablePath(providerKind.GetCommandName())))
871877
{
872878
return providerKind;

‎DotPilot.Core/ChatSessions/Execution/LlamaLocalChatClient.cs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
using System.Runtime.CompilerServices;
2+
using LLama;
3+
using LLama.Abstractions;
4+
using LLama.Common;
5+
using Microsoft.Extensions.AI;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.Extensions.Logging.Abstractions;
8+
9+
namespace DotPilot.Core.ChatSessions;
10+
11+
internal sealed class LlamaLocalChatClient : IChatClient
12+
{
13+
private readonly LLamaWeights weights;
14+
private readonly LLamaContext context;
15+
private readonly IChatClient innerChatClient;
16+
private bool disposed;
17+
18+
public LlamaLocalChatClient(
19+
string modelPath,
20+
ILogger? logger = null)
21+
{
22+
ArgumentException.ThrowIfNullOrWhiteSpace(modelPath);
23+
24+
var effectiveLogger = logger ?? NullLogger.Instance;
25+
var parameters = new ModelParams(modelPath)
26+
{
27+
ContextSize = 4096,
28+
};
29+
30+
weights = LLamaWeights.LoadFromFile(parameters);
31+
context = weights.CreateContext(parameters, effectiveLogger);
32+
var executor = new InteractiveExecutor(context, effectiveLogger);
33+
innerChatClient = executor.AsChatClient();
34+
}
35+
36+
public Task<ChatResponse> GetResponseAsync(
37+
IEnumerable<ChatMessage> messages,
38+
ChatOptions? options = null,
39+
CancellationToken cancellationToken = default)
40+
{
41+
ThrowIfDisposed();
42+
return innerChatClient.GetResponseAsync(messages, options, cancellationToken);
43+
}
44+
45+
public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
46+
IEnumerable<ChatMessage> messages,
47+
ChatOptions? options = null,
48+
CancellationToken cancellationToken = default)
49+
{
50+
ThrowIfDisposed();
51+
return StreamAsync(messages, options, cancellationToken);
52+
}
53+
54+
public object? GetService(Type serviceType, object? serviceKey = null)
55+
{
56+
ThrowIfDisposed();
57+
return innerChatClient.GetService(serviceType, serviceKey);
58+
}
59+
60+
public void Dispose()
61+
{
62+
if (disposed)
63+
{
64+
return;
65+
}
66+
67+
(innerChatClient as IDisposable)?.Dispose();
68+
context.Dispose();
69+
weights.Dispose();
70+
disposed = true;
71+
}
72+
73+
private async IAsyncEnumerable<ChatResponseUpdate> StreamAsync(
74+
IEnumerable<ChatMessage> messages,
75+
ChatOptions? options,
76+
[EnumeratorCancellation] CancellationToken cancellationToken)
77+
{
78+
await foreach (var update in innerChatClient.GetStreamingResponseAsync(messages, options, cancellationToken))
79+
{
80+
yield return update;
81+
}
82+
}
83+
84+
private void ThrowIfDisposed()
85+
{
86+
ObjectDisposedException.ThrowIf(disposed, this);
87+
}
88+
}

‎DotPilot.Core/ChatSessions/Models/AgentSessionStates.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ public enum AgentProviderKind
77
ClaudeCode,
88
GitHubCopilot,
99
Gemini,
10+
Onnx,
11+
LlamaSharp,
1012
}
1113

1214
public enum AgentProviderStatus

‎DotPilot.Core/DotPilot.Core.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
</PropertyGroup>
88

99
<ItemGroup>
10+
<PackageReference Include="LLamaSharp" />
11+
<PackageReference Include="LLamaSharp.Backend.Cpu" />
1012
<PackageReference Include="ManagedCode.ClaudeCodeSharpSDK" />
1113
<PackageReference Include="ManagedCode.ClaudeCodeSharpSDK.Extensions.AgentFramework" />
1214
<PackageReference Include="ManagedCode.ClaudeCodeSharpSDK.Extensions.AI" />
@@ -24,6 +26,7 @@
2426
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" />
2527
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" />
2628
<PackageReference Include="Microsoft.Extensions.AI" />
29+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" />
2730
<PackageReference Include="Microsoft.Orleans.Server" />
2831
</ItemGroup>
2932

‎DotPilot.Core/Providers/Configuration/AgentProviderKindExtensions.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public static string GetCommandName(this AgentProviderKind kind)
1313
AgentProviderKind.ClaudeCode => "claude",
1414
AgentProviderKind.GitHubCopilot => "copilot",
1515
AgentProviderKind.Gemini => "gemini",
16+
AgentProviderKind.Onnx => "onnx",
17+
AgentProviderKind.LlamaSharp => "llamasharp",
1618
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
1719
};
1820
}
@@ -26,6 +28,8 @@ public static string GetDefaultModelName(this AgentProviderKind kind)
2628
AgentProviderKind.ClaudeCode => "claude-sonnet-4-5",
2729
AgentProviderKind.GitHubCopilot => "gpt-5",
2830
AgentProviderKind.Gemini => GeminiModels.Gemini25Pro,
31+
AgentProviderKind.Onnx => "phi-4-mini-instruct",
32+
AgentProviderKind.LlamaSharp => "llama-3.2-3b-instruct",
2933
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
3034
};
3135
}
@@ -39,6 +43,8 @@ public static string GetDisplayName(this AgentProviderKind kind)
3943
AgentProviderKind.ClaudeCode => "Claude Code",
4044
AgentProviderKind.GitHubCopilot => "GitHub Copilot",
4145
AgentProviderKind.Gemini => "Gemini",
46+
AgentProviderKind.Onnx => "ONNX Runtime GenAI",
47+
AgentProviderKind.LlamaSharp => "LLamaSharp",
4248
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
4349
};
4450
}
@@ -52,6 +58,8 @@ public static string GetInstallCommand(this AgentProviderKind kind)
5258
AgentProviderKind.ClaudeCode => "npm install -g @anthropic-ai/claude-code",
5359
AgentProviderKind.GitHubCopilot => "npm install -g @github/copilot",
5460
AgentProviderKind.Gemini => "npm install -g @google/gemini-cli",
61+
AgentProviderKind.Onnx => kind.GetModelPathSetupCommand(),
62+
AgentProviderKind.LlamaSharp => kind.GetModelPathSetupCommand(),
5563
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
5664
};
5765
}
@@ -70,6 +78,8 @@ public static IReadOnlyList<string> GetSupportedModelNames(this AgentProviderKin
7078
GeminiModels.Gemini25Flash,
7179
GeminiModels.Gemini25FlashLite,
7280
],
81+
AgentProviderKind.Onnx => [],
82+
AgentProviderKind.LlamaSharp => [],
7383
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
7484
};
7585
}

‎DotPilot.Core/Providers/Configuration/AgentProviderKindLocalModelExtensions.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
namespace DotPilot.Core.Providers;
2+
3+
internal static class AgentProviderKindLocalModelExtensions
4+
{
5+
public static bool IsLocalModelProvider(this AgentProviderKind kind)
6+
{
7+
return kind is AgentProviderKind.Onnx or AgentProviderKind.LlamaSharp;
8+
}
9+
10+
public static IReadOnlyList<string> GetModelPathEnvironmentVariableNames(this AgentProviderKind kind)
11+
{
12+
return kind switch
13+
{
14+
AgentProviderKind.Onnx => ["DOTPILOT_ONNX_MODEL_PATH", "ONNX_MODEL_PATH"],
15+
AgentProviderKind.LlamaSharp => ["DOTPILOT_LLAMASHARP_MODEL_PATH", "LLAMASHARP_MODEL_PATH"],
16+
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
17+
};
18+
}
19+
20+
public static string GetPrimaryModelPathEnvironmentVariableName(this AgentProviderKind kind)
21+
{
22+
return kind.GetModelPathEnvironmentVariableNames()[0];
23+
}
24+
25+
public static string GetModelPathSetupCommand(this AgentProviderKind kind)
26+
{
27+
return kind switch
28+
{
29+
AgentProviderKind.Onnx => "DOTPILOT_ONNX_MODEL_PATH=/absolute/path/to/onnx-model-directory",
30+
AgentProviderKind.LlamaSharp => "DOTPILOT_LLAMASHARP_MODEL_PATH=/absolute/path/to/model.gguf",
31+
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
32+
};
33+
}
34+
35+
public static string GetLocalModelSetupSummary(this AgentProviderKind kind)
36+
{
37+
return kind switch
38+
{
39+
AgentProviderKind.Onnx => "Set the ONNX model directory path and refresh settings.",
40+
AgentProviderKind.LlamaSharp => "Set the GGUF model file path and refresh settings.",
41+
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
42+
};
43+
}
44+
45+
public static string GetLocalModelMissingSummary(this AgentProviderKind kind)
46+
{
47+
return kind switch
48+
{
49+
AgentProviderKind.Onnx => "ONNX model directory is not configured or is missing genai_config.json.",
50+
AgentProviderKind.LlamaSharp => "LLamaSharp GGUF model file is not configured or cannot be found.",
51+
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
52+
};
53+
}
54+
55+
public static string GetLocalModelReadySummary(this AgentProviderKind kind)
56+
{
57+
return kind switch
58+
{
59+
AgentProviderKind.Onnx => "Local ONNX model is ready for desktop execution.",
60+
AgentProviderKind.LlamaSharp => "Local LLamaSharp model is ready for desktop execution.",
61+
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
62+
};
63+
}
64+
}

0 commit comments

Comments
 (0)