Skip to content

Commit 8fd5ac1

Browse files
iceljciceljc
authored
Merge pull request #918 from iceljc/master
refine instruction log
2 parents 2453368 + 1275053 commit 8fd5ac1

21 files changed

Lines changed: 361 additions & 34 deletions

File tree

‎src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@ public interface IConversationService
2727
/// <param name="newMessageId">If not null, delete messages while input a new message; otherwise delete messages only</param>
2828
/// <returns></returns>
2929
Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null);
30-
Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
31-
Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
3230

3331
/// <summary>
3432
/// Send message to LLM
@@ -72,4 +70,6 @@ Task<bool> SendMessage(string agentId,
7270
/// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
7371
/// <returns></returns>
7472
Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false);
73+
74+
Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10);
7575
}

‎src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
using System.Collections.Generic;
2-
31
namespace BotSharp.Abstraction.Instructs.Models;
42

53
public class InstructLogFilter : Pagination
@@ -8,6 +6,7 @@ public class InstructLogFilter : Pagination
86
public List<string>? Providers { get; set; }
97
public List<string>? Models { get; set; }
108
public List<string>? TemplateNames { get; set; }
9+
public List<string>? UserIds { get; set; }
1110

1211
public static InstructLogFilter Empty()
1312
{

‎src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ public class InstructionLogModel
3939
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
4040
public string? UserId { get; set; }
4141

42+
[JsonIgnore]
43+
public string? UserName { get; set; }
44+
4245
[JsonIgnore]
4346
public Dictionary<string, string> States { get; set; } = [];
4447

‎src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using BotSharp.Abstraction.Instructs.Models;
2+
using BotSharp.Abstraction.Loggers.Models;
3+
4+
namespace BotSharp.Abstraction.Loggers.Services;
5+
6+
public interface ILoggerService
7+
{
8+
#region Conversation
9+
Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
10+
Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
11+
#endregion
12+
13+
#region Instruction
14+
Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter);
15+
#endregion
16+
}

‎src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,10 @@ List<string> TruncateConversation(string conversationId, string messageId, bool
151151
=> throw new NotImplementedException();
152152
List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
153153
=> throw new NotImplementedException();
154+
List<string> GetConversationsToMigrate(int batchSize = 100)
155+
=> throw new NotImplementedException();
156+
bool MigrateConvsersationLatestStates(string conversationId)
157+
=> throw new NotImplementedException();
154158
#endregion
155159

156160
#region LLM Completion Log

‎src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<TargetFramework>$(TargetFramework)</TargetFramework>

‎src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using NetTopologySuite.Algorithm;
2+
using System.Diagnostics;
3+
4+
namespace BotSharp.Core.Conversations.Services;
5+
6+
public partial class ConversationService
7+
{
8+
public async Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10)
9+
{
10+
var db = _services.GetRequiredService<IBotSharpRepository>();
11+
var isSuccess = true;
12+
var errorCount = 0;
13+
var batchNum = 0;
14+
var info = string.Empty;
15+
var error = string.Empty;
16+
17+
#if DEBUG
18+
Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n");
19+
#else
20+
_logger.LogInformation($"#Start migrating Conversation Latest States...");
21+
#endif
22+
var sw = Stopwatch.StartNew();
23+
24+
var convIds = db.GetConversationsToMigrate(batchSize);
25+
26+
while (!convIds.IsNullOrEmpty())
27+
{
28+
batchNum++;
29+
var innerSw = Stopwatch.StartNew();
30+
#if DEBUG
31+
Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n");
32+
#else
33+
_logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})");
34+
#endif
35+
36+
for (int i = 0; i < convIds.Count; i++)
37+
{
38+
var convId = convIds.ElementAt(i);
39+
try
40+
{
41+
var done = db.MigrateConvsersationLatestStates(convId);
42+
info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!";
43+
#if DEBUG
44+
Console.WriteLine($"\r\n{info}\r\n");
45+
#else
46+
_logger.LogInformation($"{info}");
47+
#endif
48+
}
49+
catch (Exception ex)
50+
{
51+
errorCount++;
52+
error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}";
53+
#if DEBUG
54+
Console.WriteLine($"\r\n{error}\r\n");
55+
#else
56+
_logger.LogError($"{error}");
57+
#endif
58+
}
59+
}
60+
61+
if (errorCount >= errorLimit)
62+
{
63+
error = $"\r\nErrors exceed limit => stop the migration!\r\n";
64+
#if DEBUG
65+
Console.WriteLine($"{error}");
66+
#else
67+
_logger.LogError($"{error}");
68+
#endif
69+
innerSw.Stop();
70+
isSuccess = false;
71+
break;
72+
}
73+
74+
innerSw.Stop();
75+
info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " +
76+
$"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)";
77+
#if DEBUG
78+
Console.WriteLine($"\r\n{info}\r\n");
79+
#else
80+
_logger.LogInformation($"{info}");
81+
#endif
82+
83+
await Task.Delay(100);
84+
convIds = db.GetConversationsToMigrate(batchSize);
85+
}
86+
87+
sw.Stop();
88+
info = $"#Done with migrating Conversation Latest States! " +
89+
$"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)";
90+
#if DEBUG
91+
Console.WriteLine($"\r\n{info}\r\n");
92+
#else
93+
_logger.LogInformation($"{info}");
94+
#endif
95+
96+
return isSuccess;
97+
}
98+
}

‎src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using BotSharp.Abstraction.Plugins.Models;
2-
using BotSharp.Abstraction.Users.Enums;
32
using Microsoft.Extensions.Configuration;
43

54
namespace BotSharp.Core.Instructs;
@@ -22,8 +21,8 @@ public bool AttachMenu(List<PluginMenuDef> menu)
2221
{
2322
SubMenu = new List<PluginMenuDef>
2423
{
25-
new PluginMenuDef("Instruction", link: "page/instruction"),
26-
new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] }
24+
new PluginMenuDef("Testing", link: "page/instruction/testing"),
25+
new PluginMenuDef("Log", link: "page/instruction/log")
2726
}
2827
});
2928

‎src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using Microsoft.Extensions.Configuration;
2+
3+
namespace BotSharp.Core.Loggers;
4+
5+
public class LoggerPlugin : IBotSharpPlugin
6+
{
7+
public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea";
8+
public string Name => "Logger";
9+
public string Description => "Provide log service";
10+
11+
public void RegisterDI(IServiceCollection services, IConfiguration config)
12+
{
13+
services.AddScoped<ILoggerService, LoggerService>();
14+
}
15+
}

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs renamed to src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
using BotSharp.Abstraction.Loggers.Models;
2-
using BotSharp.Abstraction.Repositories;
32

4-
namespace BotSharp.Core.Conversations.Services;
3+
namespace BotSharp.Core.Loggers.Services;
54

6-
public partial class ConversationService
5+
public partial class LoggerService
76
{
87
public async Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId)
98
{

0 commit comments

Comments
 (0)