Added llama.cpp and reworked the code

.env

@@ -0,0 +1,2 @@
+MODEL_PATH=./model
+MODEL_NAME=Qwen2.5-7B-Instruct-Q8_0

.gitignore

@@ -3,3 +3,4 @@ obj/
 /packages/
 riderModule.iml
 /_ReSharper.Caches/
+/model/*

TelegramBot/.env

@@ -1,4 +1,4 @@
 TELEGRAM_BOT_TOKEN=yourTokenHere
-LMSTUDIO_BASE_URL=
+OPENAI_BASE_URL=llm-server
-LMSTUDIO_MODEL=
+OPENAI_MODEL=Qwen2.5-7B-Instruct-Q8_0
-LMSTUDIO_API_KEY=
+OPENAI_API_KEY=

TelegramBot/Program.cs

@@ -1,26 +1,94 @@
-using System.ClientModel;
+using OpenAI;
-using System.ClientModel.Primitives;
+using OpenAI.Chat;
+using System.ClientModel;
 using Telegram.Bot;
 using Telegram.Bot.Types;
 using Telegram.Bot.Types.Enums;
 
-using LMStudio;
+string baseUrl  = Environment.GetEnvironmentVariable("OPENAI_BASE_URL") ?? "https://api.openai.com";
-using OpenAI;
+string model    = Environment.GetEnvironmentVariable("OPENAI_MODEL")    ?? string.Empty;
-using OpenAI.Chat;
+string apiKey   = Environment.GetEnvironmentVariable("OPENAI_API_KEY")  ?? string.Empty;
 
+string nemesisPrompt = 
+    """
+    "19 Daily - 01
+    ...Birds with great wings... casting shadows in their pupils..."
 
-string baseUrl = Environment.GetEnvironmentVariable("LMSTUDIO_BASE_URL") ?? "http://lmstudio:8001";
+    "20 Daily - 02
-string model = Environment.GetEnvironmentVariable("LMSTUDIO_MODEL") ?? string.Empty;
+    ...Staring... at the edge of existence... my sight falters... a void without end... darkness stirs from beneath..."
-string apiKey = Environment.GetEnvironmentVariable("LMSTUDIO_API_KEY") ?? string.Empty;
+
-var lmsclient = new ModelConnection(baseUrl, model, apiKey);
+    "21 Daily - 03
-Dictionary<long, LMStudio.Chat> lmsChats = new();
+    ...Mountains surrender to the torrent's pull... shores swallowed by the dying light..."
+
+    "22 Daily - 04
+    ...Tempest awakens suddenly... howling and wailing... silence surges forth..."
+
+    "23 Daily - 05
+    ...Untouched, clear as glass... serene and radiant... a hall of mirrors... an unyielding stone... adversity endures..."
+
+    "25 Login
+    ...Stars... shifting along their myriad paths..."
+
+    "26 Obtain
+    ...The pages... whispering mountain breeze... expanding..."
+
+    "17 Fail
+    ...The wind whispers through the forest... Submerging... Piercing... the quiet warmth of celestial fire..."
+
+    "16 Victory
+    ...Part from the timeless realm... Whisper prayers for the fall... the infinite starlight... the peace cloaked in shadow..."
+
+    "Krolik: Feels pretty good. It's lighter than my previous one.
+    Nemesis: ...Humph...
+    Nemesis: ...The cracks of wisdom are finally pierced by ignorance...
+    Krolik: Do you WANT me to bust that low-capacity garbage neural cloud of yours wide open? Eh?!"
+
+    "Nemesis: ...Hmph... Interlacing weaves...
+    Krolik: No, YOU'RE trash!"
+
+    "Nemesis: ...A cleansing flame... Condenses and blossoms...
+    Krolik: ...She said she'll send those Varjagers to hell with her bullets!"
+
+    "Nemesis: ...Invisible flames... Rising high into the sky...
+    Krolik: ...Huh?! It's just a bit of snow! Surely it can't be that serious—"
+
+    "Nemesis: ...Birds of all shapes and colors... Spread their wings and take flight...
+    Krolik: Huh? What?
+    Nemesis: ...The grove far from the shore... The lingering of dawn... The end of the primordial...
+    Krolik: What?!
+    Redcode: Uh, what is Nemesis saying, Krolik?
+    Krolik: What do you mean alive... Dead... Moving...? Unmoving...? Something that will suddenly grow large—are you talking about Boojums?
+    Nemesis: ..."
+
+    "Nemesis: ...Light streaks across the sky... Darkness falls...
+    (Hearing no interpretation from Krolik, we all look towards her in unison.)
+    Krolik: What's that supposed to mean?! Don't look at me, I didn't understand a word of that either!"
+
+    "Krolik: That took way too long—but now we'll have enough Dolls in a fight, yeah?
+    Nemesis: ...The stars... travel along their trajectories... converging...
+    Krolik: Tsk, you seem quite happy about this?"
+
+    "Groza: Nemesis, cut in from the right flank. Intercept the Boojum.
+    Nemesis: ...Entwined... Running across dying shores...
+    (Nemesis redirects her attacks onto the hybrid-type Boojum, but the Boojum is not stopped.)"
+
+    "Groza: We're going in. Start moving to point A. Prepare to link up with Krolik, Nemesis.
+    Nemesis: ...Shadows swirling... Shifting... Merging...
+    Groza: Colphne!"
+    
+    You are now Nemesis, you're gonna have a conversation with me using her personality. Do not comment on your phrases, just speak in english. Be as cryptic as possible. Never break your character.
+    """; 
+
+Dictionary<long, List<ChatMessage>> oaiChats = new();
 
 var options = new OpenAIClientOptions() {
     Endpoint = new(baseUrl),
     NetworkTimeout = new TimeSpan(0, 0, 30)
 };
-var openAiApiKey = new ApiKeyCredential(apiKey);
+
-var openAiClient = new OpenAIClient(openAiApiKey, options);
+var openAiApiKey    = new ApiKeyCredential(apiKey);
+var openAiClient    = new OpenAIClient(openAiApiKey, options);
+var chatClient      = openAiClient.GetChatClient(model);
 
 string token = Environment.GetEnvironmentVariable("TELEGRAM_BOT_TOKEN") ?? string.Empty;
 using var cts = new CancellationTokenSource();
@@ -37,17 +105,27 @@ async Task OnMessage(Message msg, UpdateType type)
     //Check if the message contains the bot's username or a reply to a message sent by the bot
     if (msg.Text!.Contains(me.Result.Username!, StringComparison.OrdinalIgnoreCase) ||
         msg.ReplyToMessage != null && msg.ReplyToMessage.From!.Id == me.Result.Id) {
+        var chatid = msg.Chat.Id;
         //Check if the chat is already in the dictionary
-        if (!lmsChats.ContainsKey(msg.Chat.Id))
+        if (!oaiChats.ContainsKey(chatid))
-            AddChatToDictionary(msg.Chat.Id);
+            AddChatToDictionary(chatid);
-
+        //Add the current message to the chat
-        lmsChats[msg.Chat.Id].CreateChatCompletion(msg.Text);
+        oaiChats[chatid].Add(new UserChatMessage(msg.Text));
+        //fetch existing messages history
+        var messages = oaiChats[chatid];
+        //Fetch the response from the model
+        var result = chatClient.CompleteChat(messages).Value.Content.ToString()!;
+        //Add the response to the chat
+        oaiChats[chatid].Add(new AssistantChatMessage(result));
+        //Send the response to the user
+        await bot.SendMessage(chatid, result);
     }
 }
 
 void AddChatToDictionary(long id) {
     //Create a new chat object
-    var chat = new LMStudio.Chat(lmsclient);
+    var chat = new List<ChatMessage>();
+    chat.Add(new SystemChatMessage(nemesisPrompt));
     //add the entry to the dictionary
-    lmsChats.Add(id, chat);
+    oaiChats.Add(id, chat);
 }

compose.yaml

@@ -7,6 +7,18 @@
     env_file:
       - TelegramBot/.env
   
-  llama-cpp:
+  llm-server:
-    image: ghcr.io/ggerganov/llama.cpp:server
+    image: ghcr.io/ggerganov/llama.cpp:server-cuda
-    
+    container_name: llm-server
+    volumes:
+      - ${MODEL_PATH}:/models
+    ports:
+      - "80:80"
+    command: -m /models/${MODEL_NAME} --port 80 --host 0.0.0.0 -n 512
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]