organic-CPU/GPU-Recource-Manager-and-proxy-for-ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added support to unload ollama model when client requests.

Browse Source
This commit is contained in:
mrmarcus007
2025-11-14 16:05:15 +01:00
parent 8282740de4
commit 2fbc1ffa6f
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
GPU Resource Manager Proxy for Ollama (Designed for Proxmox).py
+6
View File
@@ -314,6 +314,12 @@ class OllamaProxyHandler(http.server.SimpleHTTPRequestHandler):
self.send_error(500, f"Internal server error: {e}") self.send_error(500, f"Internal server error: {e}")
def _is_gpu_intensive_operation(self, path, request_data): def _is_gpu_intensive_operation(self, path, request_data):
if path == '/api/generate' and request_data.get('keep_alive') == 0:
logging.debug("Detected model unload request via /api/generate")
return False
if path == '/api/chat' and request_data.get('keep_alive') == 0 and request_data.get('messages') == []:
logging.debug("Detected model unload request via /api/chat")
return False
if path in ['/api/generate', '/api/chat', '/api/embeddings']: if path in ['/api/generate', '/api/chat', '/api/embeddings']:
return True return True
if path == '/api/load': if path == '/api/load':