Skip to content

Commit 2453368

Browse files
authored
Merge pull request #919 from hchen2020/master
Seperate realtime in a new project
2 parents 4dadd5c + 28d4e32 commit 2453368

14 files changed

Lines changed: 220 additions & 110 deletions

File tree

BotSharp.sln

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\
127127
EndProject
128128
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}"
129129
EndProject
130+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}"
131+
EndProject
130132
Global
131133
GlobalSection(SolutionConfigurationPlatforms) = preSolution
132134
Debug|Any CPU = Debug|Any CPU
@@ -519,6 +521,14 @@ Global
519521
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|Any CPU.Build.0 = Release|Any CPU
520522
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.ActiveCfg = Release|Any CPU
521523
{AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU
524+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
525+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
526+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.ActiveCfg = Debug|Any CPU
527+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Debug|x64.Build.0 = Debug|Any CPU
528+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
529+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|Any CPU.Build.0 = Release|Any CPU
530+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.ActiveCfg = Release|Any CPU
531+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4}.Release|x64.Build.0 = Release|Any CPU
522532
EndGlobalSection
523533
GlobalSection(SolutionProperties) = preSolution
524534
HideSolutionNode = FALSE
@@ -580,6 +590,7 @@ Global
580590
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
581591
{AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
582592
{AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
593+
{7ACD8C95-C66B-436A-80E7-541A57D8C3F4} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
583594
EndGlobalSection
584595
GlobalSection(ExtensibilityGlobals) = postSolution
585596
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

Directory.Packages.props

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
1414
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
1515
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
16-
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
16+
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
1717
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
1818
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
1919
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
@@ -29,7 +29,7 @@
2929
<PackageVersion Include="Nanoid" Version="3.1.0" />
3030
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
3131
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
32-
<PackageVersion Include="Anthropic.SDK" Version="4.4.2" />
32+
<PackageVersion Include="Anthropic.SDK" Version="4.7.1" />
3333
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
3434
<PackageVersion Include="NAudio" Version="2.2.1" />
3535
<PackageVersion Include="NAudio.Core" Version="2.2.1" />
@@ -43,7 +43,7 @@
4343
<PackageVersion Include="MySql.Data" Version="9.0.0" />
4444
<PackageVersion Include="NPOI" Version="2.7.1" />
4545
<PackageVersion Include="LLMSharp.Google.Palm" Version="1.0.2" />
46-
<PackageVersion Include="Mscc.GenerativeAI" Version="2.0.1" />
46+
<PackageVersion Include="Mscc.GenerativeAI" Version="2.2.11" />
4747
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
4848
<PackageVersion Include="Refit" Version="8.0.0" />
4949
<PackageVersion Include="Refit.HttpClientFactory" Version="8.0.0" />
@@ -67,7 +67,7 @@
6767
<PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
6868
<PackageVersion Include="System.Drawing.Common" Version="8.0.7" />
6969
<PackageVersion Include="pythonnet" Version="3.0.4" />
70-
<PackageVersion Include="Qdrant.Client" Version="1.11.0" />
70+
<PackageVersion Include="Qdrant.Client" Version="1.13.0" />
7171
<PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
7272
<PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
7373
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using BotSharp.Abstraction.MLTasks;
12
using BotSharp.Abstraction.Realtime.Models;
23
using System.Net.WebSockets;
34

@@ -8,5 +9,11 @@ namespace BotSharp.Abstraction.Realtime;
89
/// </summary>
910
public interface IRealtimeHub
1011
{
11-
Task Listen(WebSocket userWebSocket, Func<string, RealtimeHubConnection> onUserMessageReceived);
12+
RealtimeHubConnection HubConn { get; }
13+
RealtimeHubConnection SetHubConnection(string conversationId);
14+
15+
IRealTimeCompletion Completer { get; }
16+
IRealTimeCompletion SetCompleter(string provider);
17+
18+
Task Listen(WebSocket userWebSocket, Action<string> onUserMessageReceived);
1219
}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ public class RealtimeHubConnection
88
public string StreamId { get; set; } = null!;
99
public string? LastAssistantItemId { get; set; } = null!;
1010
public long LatestMediaTimestamp { get; set; }
11-
public long? ResponseStartTimestampTwilio { get; set; }
11+
public long? ResponseStartTimestamp { get; set; }
1212
public string KeypadInputBuffer { get; set; } = string.Empty;
1313
public ConcurrentQueue<string> MarkQueue { get; set; } = new();
1414
public string CurrentAgentId { get; set; } = null!;
@@ -23,12 +23,12 @@ public void ResetResponseState()
2323
{
2424
MarkQueue.Clear();
2525
LastAssistantItemId = null;
26-
ResponseStartTimestampTwilio = null;
26+
ResponseStartTimestamp = null;
2727
}
2828

2929
public void ResetStreamState()
3030
{
31-
ResponseStartTimestampTwilio = null;
31+
ResponseStartTimestamp = null;
3232
LatestMediaTimestamp = 0;
3333
}
3434
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
11+
</ItemGroup>
12+
13+
</Project>
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
using BotSharp.Abstraction.Utilities;
2+
3+
namespace BotSharp.Core.Realtime.Hooks;
4+
5+
public class RealtimeConversationHook : ConversationHookBase, IConversationHook
6+
{
7+
private readonly IServiceProvider _services;
8+
public RealtimeConversationHook(IServiceProvider services)
9+
{
10+
_services = services;
11+
}
12+
13+
public async Task OnFunctionExecuting(RoleDialogModel message)
14+
{
15+
var hub = _services.GetRequiredService<IRealtimeHub>();
16+
if (hub.HubConn == null)
17+
{
18+
return;
19+
}
20+
// Save states
21+
var states = _services.GetRequiredService<IConversationStateService>();
22+
states.SaveStateByArgs(message.FunctionArgs?.JsonContent<JsonDocument>() ?? JsonDocument.Parse("{}"));
23+
}
24+
25+
public async Task OnFunctionExecuted(RoleDialogModel message)
26+
{
27+
var hub = _services.GetRequiredService<IRealtimeHub>();
28+
if (hub.HubConn == null)
29+
{
30+
return;
31+
}
32+
var routing = _services.GetRequiredService<IRoutingService>();
33+
34+
message.Role = AgentRole.Function;
35+
36+
if (message.FunctionName == "route_to_agent")
37+
{
38+
var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}") ?? new();
39+
message.Content = $"Connected to agent of {inst.AgentName}";
40+
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
41+
42+
await hub.Completer.UpdateSession(hub.HubConn);
43+
await hub.Completer.InsertConversationItem(message);
44+
await hub.Completer.TriggerModelInference($"Guide the user through the next steps of the process as this Agent ({inst.AgentName}), following its instructions and operational procedures.");
45+
}
46+
else if (message.FunctionName == "util-routing-fallback_to_router")
47+
{
48+
var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}") ?? new();
49+
message.Content = $"Returned to Router due to {inst.Reason}";
50+
hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
51+
52+
await hub.Completer.UpdateSession(hub.HubConn);
53+
await hub.Completer.InsertConversationItem(message);
54+
await hub.Completer.TriggerModelInference($"Check with user whether to proceed the new request: {inst.Reason}");
55+
}
56+
else
57+
{
58+
// Update session for changed states
59+
await hub.Completer.UpdateSession(hub.HubConn);
60+
await hub.Completer.InsertConversationItem(message);
61+
await hub.Completer.TriggerModelInference("Reply based on the function's output.");
62+
}
63+
}
64+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
using BotSharp.Abstraction.Plugins;
2+
using BotSharp.Core.Realtime.Hooks;
3+
using BotSharp.Core.Realtime.Services;
4+
using Microsoft.Extensions.Configuration;
5+
6+
namespace BotSharp.Core.Realtime;
7+
8+
public class RealtimePlugin : IBotSharpPlugin
9+
{
10+
public string Id => "68c1c737-5c21-49de-b141-cd5c8d9bf978";
11+
public string Name => "Realtime Hub";
12+
public string? IconUrl => "https://thumbs.dreamstime.com/b/microphone-icon-sound-waves-voice-command-recording-message-sign-349007898.jpg";
13+
public string Description => "Build low-latency, multi-modal experiences with the Realtime API.";
14+
15+
public void RegisterDI(IServiceCollection services, IConfiguration config)
16+
{
17+
services.AddScoped<IRealtimeHub, RealtimeHub>();
18+
services.AddScoped<IConversationHook, RealtimeConversationHook>();
19+
}
20+
}

0 commit comments

Comments
 (0)