using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MercatorUi;
using MercatorApi;
using MercatorExtensions;
// <CompileWithRoslyn />
namespace MyAssembly
{
public static class ChatGPT
{
private const string tutosCollectionName = "Tutos";
public async static Task VectorizeTutos()
{
if (!Dialogs.AnswerYesNo("De inhoud van de tutorials echt vectoriseren?"))
return;
MercatorAI.FactoryOpenAI factoryOpenAI = new MercatorAI.FactoryOpenAI();
MercatorAI.Interfaces.IOpenAiEmbeddingGeneratorClient openAiEmbeddingGeneratorClient = factoryOpenAI.CreateOpenAiEmbeddingGeneratorClient<string>("text-embedding-3-small", out string error);
if (!string.IsNullOrEmpty(error))
{
Dialogs.Stop("Kan de embeddings-generator van OpenAI niet initialiseren: " + error);
return;
}
MercatorAI.FactoryQdrant factoryQdrant = new MercatorAI.FactoryQdrant();
using (MercatorAI.Interfaces.IQdrantClient qdrantClient = factoryQdrant.CreateQdrantClient("http://192.168...:6334", out error))
{
if (!string.IsNullOrEmpty(error))
{
Dialogs.Stop("Kan de Qdrant-client niet initialiseren: " + error);
return;
}
MercatorUi.Wait.WaitStatic.WaitWindow(Api.Iif_langue(Globals.Langue, IifLangueEnum.DataUpdate));
await qdrantClient.CollectionDeleteAsync<Guid, DocChunk>(tutosCollectionName);
using (var embeddingGenerator = openAiEmbeddingGeneratorClient.EmbeddingGenerator)
using (var collection = await qdrantClient.CreateCollectionAsync<Guid, DocChunk>(tutosCollectionName, embeddingGenerator))
{
foreach (string docxFile in Api.GetFiles("<Other\\Tutos", "*.docx"))
{
string sourceId = Api.JustFName(docxFile);
string fullText = Api.DocxToText(docxFile);
List<string> chunks = qdrantClient.TextChunkerSplitPlainTextLines(fullText, maxTokensPerLine: 400);
IEnumerable<DocChunk> records = chunks.Select((text, i) => new DocChunk
{
Chunk = i,
SourceId = sourceId,
Content = text
});
foreach (DocChunk[] arrayOfDocChunks in records.Chunk(500)) // we verzenden per blok van 500
{
await qdrantClient.CollectionUpsertAsync<Guid, DocChunk>(collection, arrayOfDocChunks);
}
}
}
MercatorUi.Wait.WaitStatic.WaitClear();
Dialogs.Stop(Api.Iif_langue(Globals.Langue, IifLangueEnum.UpdateAchieved));
}
}
}
}