Skip to content

Commit 96e6372

Browse files
SommerEngineeringSommerEngineering
authored
Added a common augmentation interface & default implementation (#288)
1 parent f01cf49 commit 96e6372

19 files changed

+249
-62
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ Things we are currently working on:
2020
- [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb)
2121
- [ ] App: Implement the continuous process of vectorizing data
2222
- [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~
23-
- [ ] App: Define a common augmentation interface for the integration of RAG processes in chats
23+
- [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288))~~
2424
- [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~
2525

2626

‎app/MindWork AI Studio.sln.DotSettings

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MSG/@EntryIndexedValue">MSG</s:String>
88
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RAG/@EntryIndexedValue">RAG</s:String>
99
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UI/@EntryIndexedValue">UI</s:String>
10+
<s:Boolean x:Key="/Default/UserDictionary/Words/=agentic/@EntryIndexedValue">True</s:Boolean>
1011
<s:Boolean x:Key="/Default/UserDictionary/Words/=groq/@EntryIndexedValue">True</s:Boolean>
1112
<s:Boolean x:Key="/Default/UserDictionary/Words/=ollama/@EntryIndexedValue">True</s:Boolean>
1213
<s:Boolean x:Key="/Default/UserDictionary/Words/=tauri_0027s/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

‎app/MindWork AI Studio/Chat/ChatRole.cs

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public enum ChatRole
1212
USER,
1313
AI,
1414
AGENT,
15+
RAG,
1516
}
1617

1718
/// <summary>

‎app/MindWork AI Studio/Chat/ContentImage.cs

+1-59
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ namespace AIStudio.Chat;
77
/// <summary>
88
/// Represents an image inside the chat.
99
/// </summary>
10-
public sealed class ContentImage : IContent
10+
public sealed class ContentImage : IContent, IImageSource
1111
{
1212
#region Implementation of IContent
1313

@@ -47,62 +47,4 @@ public Task CreateFromProviderAsync(IProvider provider, Model chatModel, IConten
4747
/// The image source.
4848
/// </summary>
4949
public required string Source { get; set; }
50-
51-
/// <summary>
52-
/// Read the image content as a base64 string.
53-
/// </summary>
54-
/// <remarks>
55-
/// The images are directly converted to base64 strings. The maximum
56-
/// size of the image is around 10 MB. If the image is larger, the method
57-
/// returns an empty string.
58-
///
59-
/// As of now, this method does no sort of image processing. LLMs usually
60-
/// do not work with arbitrary image sizes. In the future, we might have
61-
/// to resize the images before sending them to the model.
62-
/// </remarks>
63-
/// <param name="token">The cancellation token.</param>
64-
/// <returns>The image content as a base64 string; might be empty.</returns>
65-
public async Task<string> AsBase64(CancellationToken token = default)
66-
{
67-
switch (this.SourceType)
68-
{
69-
case ContentImageSource.BASE64:
70-
return this.Source;
71-
72-
case ContentImageSource.URL:
73-
{
74-
using var httpClient = new HttpClient();
75-
using var response = await httpClient.GetAsync(this.Source, HttpCompletionOption.ResponseHeadersRead, token);
76-
if(response.IsSuccessStatusCode)
77-
{
78-
// Read the length of the content:
79-
var lengthBytes = response.Content.Headers.ContentLength;
80-
if(lengthBytes > 10_000_000)
81-
return string.Empty;
82-
83-
var bytes = await response.Content.ReadAsByteArrayAsync(token);
84-
return Convert.ToBase64String(bytes);
85-
}
86-
87-
return string.Empty;
88-
}
89-
90-
case ContentImageSource.LOCAL_PATH:
91-
if(File.Exists(this.Source))
92-
{
93-
// Read the content length:
94-
var length = new FileInfo(this.Source).Length;
95-
if(length > 10_000_000)
96-
return string.Empty;
97-
98-
var bytes = await File.ReadAllBytesAsync(this.Source, token);
99-
return Convert.ToBase64String(bytes);
100-
}
101-
102-
return string.Empty;
103-
104-
default:
105-
return string.Empty;
106-
}
107-
}
10850
}

‎app/MindWork AI Studio/Chat/IImageSource.cs

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace AIStudio.Chat;
2+
3+
public interface IImageSource
4+
{
5+
/// <summary>
6+
/// The type of the image source.
7+
/// </summary>
8+
/// <remarks>
9+
/// Is the image source a URL, a local file path, a base64 string, etc.?
10+
/// </remarks>
11+
public ContentImageSource SourceType { get; init; }
12+
13+
/// <summary>
14+
/// The image source.
15+
/// </summary>
16+
public string Source { get; set; }
17+
}

‎app/MindWork AI Studio/Chat/IImageSourceExtensions.cs

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
namespace AIStudio.Chat;
2+
3+
public static class IImageSourceExtensions
4+
{
5+
/// <summary>
6+
/// Read the image content as a base64 string.
7+
/// </summary>
8+
/// <remarks>
9+
/// The images are directly converted to base64 strings. The maximum
10+
/// size of the image is around 10 MB. If the image is larger, the method
11+
/// returns an empty string.
12+
///
13+
/// As of now, this method does no sort of image processing. LLMs usually
14+
/// do not work with arbitrary image sizes. In the future, we might have
15+
/// to resize the images before sending them to the model.
16+
/// </remarks>
17+
/// <param name="image">The image source.</param>
18+
/// <param name="token">The cancellation token.</param>
19+
/// <returns>The image content as a base64 string; might be empty.</returns>
20+
public static async Task<string> AsBase64(this IImageSource image, CancellationToken token = default)
21+
{
22+
switch (image.SourceType)
23+
{
24+
case ContentImageSource.BASE64:
25+
return image.Source;
26+
27+
case ContentImageSource.URL:
28+
{
29+
using var httpClient = new HttpClient();
30+
using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, token);
31+
if(response.IsSuccessStatusCode)
32+
{
33+
// Read the length of the content:
34+
var lengthBytes = response.Content.Headers.ContentLength;
35+
if(lengthBytes > 10_000_000)
36+
return string.Empty;
37+
38+
var bytes = await response.Content.ReadAsByteArrayAsync(token);
39+
return Convert.ToBase64String(bytes);
40+
}
41+
42+
return string.Empty;
43+
}
44+
45+
case ContentImageSource.LOCAL_PATH:
46+
if(File.Exists(image.Source))
47+
{
48+
// Read the content length:
49+
var length = new FileInfo(image.Source).Length;
50+
if(length > 10_000_000)
51+
return string.Empty;
52+
53+
var bytes = await File.ReadAllBytesAsync(image.Source, token);
54+
return Convert.ToBase64String(bytes);
55+
}
56+
57+
return string.Empty;
58+
59+
default:
60+
return string.Empty;
61+
}
62+
}
63+
}

‎app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
3737
ChatRole.USER => "user",
3838
ChatRole.AI => "assistant",
3939
ChatRole.AGENT => "assistant",
40+
ChatRole.RAG => "assistant",
4041

4142
_ => "user",
4243
},

‎app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
4949
ChatRole.AI => "assistant",
5050
ChatRole.AGENT => "assistant",
5151
ChatRole.SYSTEM => "system",
52+
ChatRole.RAG => "assistant",
5253

5354
_ => "user",
5455
},

‎app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
5050
ChatRole.AI => "assistant",
5151
ChatRole.AGENT => "assistant",
5252
ChatRole.SYSTEM => "system",
53+
ChatRole.RAG => "assistant",
5354

5455
_ => "user",
5556
},

‎app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
5050
ChatRole.AI => "assistant",
5151
ChatRole.AGENT => "assistant",
5252
ChatRole.SYSTEM => "system",
53+
ChatRole.RAG => "assistant",
5354

5455
_ => "user",
5556
},

‎app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
4848
ChatRole.AI => "assistant",
4949
ChatRole.AGENT => "assistant",
5050
ChatRole.SYSTEM => "system",
51+
ChatRole.RAG => "assistant",
5152

5253
_ => "user",
5354
},

‎app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

+1
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
7676
ChatRole.USER => "user",
7777
ChatRole.AI => "assistant",
7878
ChatRole.AGENT => "assistant",
79+
ChatRole.RAG => "assistant",
7980
ChatRole.SYSTEM => systemPromptRole,
8081

8182
_ => "user",

‎app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
4646
ChatRole.AI => "assistant",
4747
ChatRole.AGENT => "assistant",
4848
ChatRole.SYSTEM => "system",
49+
ChatRole.RAG => "assistant",
4950

5051
_ => "user",
5152
},

‎app/MindWork AI Studio/Provider/X/ProviderX.cs

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
5050
ChatRole.AI => "assistant",
5151
ChatRole.AGENT => "assistant",
5252
ChatRole.SYSTEM => "system",
53+
ChatRole.RAG => "assistant",
5354

5455
_ => "user",
5556
},

‎app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
using System.Text;
2+
3+
using AIStudio.Chat;
4+
using AIStudio.Provider;
5+
6+
namespace AIStudio.Tools.RAG.AugmentationProcesses;
7+
8+
public sealed class AugmentationOne : IAugmentationProcess
9+
{
10+
#region Implementation of IAugmentationProcess
11+
12+
/// <inheritdoc />
13+
public string TechnicalName => "AugmentationOne";
14+
15+
/// <inheritdoc />
16+
public string UIName => "Standard augmentation process";
17+
18+
/// <inheritdoc />
19+
public string Description => "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.";
20+
21+
/// <inheritdoc />
22+
public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
23+
{
24+
var logger = Program.SERVICE_PROVIDER.GetService<ILogger<AugmentationOne>>()!;
25+
if(retrievalContexts.Count == 0)
26+
{
27+
logger.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
28+
return chatThread;
29+
}
30+
31+
var numTotalRetrievalContexts = retrievalContexts.Count;
32+
logger.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");
33+
34+
//
35+
// We build a huge prompt from all retrieval contexts:
36+
//
37+
var sb = new StringBuilder();
38+
sb.AppendLine("The following useful information will help you in processing the user prompt:");
39+
sb.AppendLine();
40+
41+
var index = 0;
42+
foreach(var retrievalContext in retrievalContexts)
43+
{
44+
index++;
45+
sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
46+
sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
47+
sb.AppendLine($"Content category: {retrievalContext.Category}");
48+
sb.AppendLine($"Content type: {retrievalContext.Type}");
49+
sb.AppendLine($"Content path: {retrievalContext.Path}");
50+
51+
if(retrievalContext.Links.Count > 0)
52+
{
53+
sb.AppendLine("Additional links:");
54+
foreach(var link in retrievalContext.Links)
55+
sb.AppendLine($"- {link}");
56+
}
57+
58+
switch(retrievalContext)
59+
{
60+
case RetrievalTextContext textContext:
61+
sb.AppendLine();
62+
sb.AppendLine("Matched text content:");
63+
sb.AppendLine("````");
64+
sb.AppendLine(textContext.MatchedText);
65+
sb.AppendLine("````");
66+
67+
if(textContext.SurroundingContent.Count > 0)
68+
{
69+
sb.AppendLine();
70+
sb.AppendLine("Surrounding text content:");
71+
foreach(var surrounding in textContext.SurroundingContent)
72+
{
73+
sb.AppendLine();
74+
sb.AppendLine("````");
75+
sb.AppendLine(surrounding);
76+
sb.AppendLine("````");
77+
}
78+
}
79+
80+
81+
break;
82+
83+
case RetrievalImageContext imageContext:
84+
sb.AppendLine();
85+
sb.AppendLine("Matched image content as base64-encoded data:");
86+
sb.AppendLine("````");
87+
sb.AppendLine(await imageContext.AsBase64(token));
88+
sb.AppendLine("````");
89+
break;
90+
91+
default:
92+
logger.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
93+
break;
94+
}
95+
96+
sb.AppendLine();
97+
}
98+
99+
//
100+
// Append the entire augmentation to the chat thread,
101+
// just before the user prompt:
102+
//
103+
chatThread.Blocks.Insert(chatThread.Blocks.Count - 1, new()
104+
{
105+
Role = ChatRole.RAG,
106+
Time = DateTimeOffset.UtcNow,
107+
ContentType = ContentType.TEXT,
108+
HideFromUser = true,
109+
Content = new ContentText
110+
{
111+
Text = sb.ToString(),
112+
}
113+
});
114+
115+
return chatThread;
116+
}
117+
118+
#endregion
119+
}

0 commit comments

Comments
 (0)