Added llama.cpp and reworked the code

2024-12-26 00:35:00 +01:00
parent c6302112b2
commit 2357c7570c
6 changed files with 118 additions and 25 deletions
--- a/compose.yaml
+++ b/compose.yaml
@@ -7,6 +7,18 @@
    env_file:
      - TelegramBot/.env
  
-  llama-cpp:
-    image: ghcr.io/ggerganov/llama.cpp:server
-    
+  llm-server:
+    image: ghcr.io/ggerganov/llama.cpp:server-cuda
+    container_name: llm-server
+    volumes:
+      - ${MODEL_PATH}:/models
+    ports:
+      - "80:80"
+    command: -m /models/${MODEL_NAME} --port 80 --host 0.0.0.0 -n 512
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]