""" 🦙 Local LLM — ComfyUI içinde GGUF model çalıştır (Ollama alternatifi) Mantık: - llama-cpp-python ile GGUF modelleri lazy-load - Tek bir model bellekte tutulur (LRU cache=1) - ComfyUI'nin Flux'unu unload ettikten sonra LLM yüklenir - LLM kullanım bittikten sonra explicit unload edilir - Her zaman aynı API: chat(system, user) → metin """ import os import gc import time import folder_paths LLM_DIR = os.path.join(folder_paths.models_dir, "LLM") os.makedirs(LLM_DIR, exist_ok=True) def list_local_models(): """models/LLM/ klasöründeki tüm GGUF dosyalarını listele.""" out = [] if not os.path.isdir(LLM_DIR): return out for root, _, files in os.walk(LLM_DIR): for f in files: if not f.lower().endswith(".gguf"): continue # mmproj dosyaları (vision projection) listede gözükmesin if "mmproj" in f.lower(): continue full = os.path.join(root, f) rel = os.path.relpath(full, LLM_DIR).replace("\\", "/") sz = os.path.getsize(full) out.append({"name": rel, "path": full, "size": sz, "size_gb": round(sz / 1024**3, 2)}) out.sort(key=lambda x: x["name"]) return out def is_available(): """llama-cpp-python yüklü mü?""" try: import llama_cpp # noqa return True except ImportError: return False class _LocalLLMSingleton: """Tek bir model bellekte tutar (LRU cache=1). Yeni model istenince eskisini unload eder.""" _instance = None def __new__(cls): if cls._instance is None: cls._instance = super().__new__(cls) cls._instance._llm = None cls._instance._loaded_path = None cls._instance._loaded_at = None cls._instance._params = {} return cls._instance @property def loaded_model(self): return self._loaded_path def _reset_state(self): """State'i temizle (crash sonrası zombie referansları engelle).""" self._llm = None self._loaded_path = None self._loaded_at = None self._params = {} gc.collect() try: import torch if torch.cuda.is_available(): torch.cuda.empty_cache() except Exception: pass def load(self, gguf_path, n_ctx=8192, n_gpu_layers=-1, verbose=False): """GGUF model yükle. Aynısı zaten yüklüyse no-op. Crash'te state temizlenir.""" if not os.path.isfile(gguf_path): raise FileNotFoundError(f"GGUF dosyası yok: {gguf_path}") # Aynı model + aynı parametreler → no-op params_now = {"n_ctx": n_ctx, "n_gpu_layers": n_gpu_layers} if (self._llm is not None and self._loaded_path == gguf_path and self._params == params_now): return # Eski modeli boşalt if self._llm is not None: self.unload() # Yeni modeli yükle try: from llama_cpp import Llama except ImportError: raise RuntimeError( "llama-cpp-python yuklu degil. " "Yukleme: python_embeded\\python.exe -s -m pip install llama-cpp-python " "--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu/" ) print(f"[FikoYT LocalLLM] Yukleniyor: {os.path.basename(gguf_path)} " f"(ctx={n_ctx}, gpu_layers={n_gpu_layers})") t0 = time.time() try: llm = Llama( model_path=gguf_path, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers, verbose=verbose, chat_format="chatml", # çoğu modern model için uyumlu ) except Exception as e: # Llama constructor crash'i (CPU/CUDA hatası, OOM vs.) — state'i temiz tut self._reset_state() raise RuntimeError(f"GGUF model yuklenemedi: {e}") from e self._llm = llm self._loaded_path = gguf_path self._loaded_at = time.time() self._params = params_now print(f"[FikoYT LocalLLM] Yüklendi ({time.time()-t0:.1f}s)") def unload(self): if self._llm is not None: print(f"[FikoYT LocalLLM] Unload: {os.path.basename(self._loaded_path or '?')}") # `Exception ignored in __del__` mesajlarını engellemek için # önce close() çağır (varsa), sonra referansı bırak. llm = self._llm self._llm = None # önce kendi referansımızı sıfırla for fn in ("close", "_close", "reset", "__del__"): try: method = getattr(llm, fn, None) if callable(method) and fn != "__del__": method() break except Exception: pass try: del llm except Exception: pass self._loaded_path = None self._loaded_at = None self._params = {} gc.collect() try: import torch if torch.cuda.is_available(): torch.cuda.empty_cache() except Exception: pass def chat(self, system_prompt, user_prompt, temperature=0.6, max_tokens=4096): """Chat completion — sistem ve kullanıcı promptu al, metin yanıt döndür. Inference crash'inde state temizlenir (zombie engeli).""" if self._llm is None: raise RuntimeError("Model yuklu degil. Once load() cagir.") messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] t0 = time.time() try: resp = self._llm.create_chat_completion( messages=messages, temperature=temperature, max_tokens=max_tokens, ) except Exception as e: # Inference crash'i — buffer'lar bozulmuş olabilir, modeli unload et print(f"[FikoYT LocalLLM] Inference crash, state temizleniyor: {e}") try: self.unload() except Exception: self._reset_state() raise dt = time.time() - t0 content = resp["choices"][0]["message"]["content"] # token sayısı bilgisi usage = resp.get("usage", {}) print(f"[FikoYT LocalLLM] {dt:.1f}s, " f"in={usage.get('prompt_tokens', '?')} out={usage.get('completion_tokens', '?')}") return content def get_singleton(): return _LocalLLMSingleton() def local_llm_chat(gguf_name_or_path, system_prompt, user_prompt, temperature=0.6, max_tokens=4096, n_ctx=8192, n_gpu_layers=-1, auto_unload=False): """Tek-shot chat — model yükler (yüklü değilse), yanıt verir, opsiyonel unload. gguf_name_or_path: 'models/LLM/''a göre rel path ya da tam path. """ if not is_available(): raise RuntimeError( "llama-cpp-python yuklu degil. Settings'te 'LLM Engine' altinda " "'⚡ Otomatik Yukle' butonuna basabilirsin." ) # Path normalize if os.path.isabs(gguf_name_or_path): gguf_path = gguf_name_or_path else: gguf_path = os.path.join(LLM_DIR, gguf_name_or_path) if not os.path.isfile(gguf_path): # Direkt models/LLM içinde bul for m in list_local_models(): if m["name"].endswith(gguf_name_or_path) or os.path.basename(m["name"]) == gguf_name_or_path: gguf_path = m["path"] break else: raise FileNotFoundError(f"GGUF bulunamadi: {gguf_name_or_path}") inst = get_singleton() inst.load(gguf_path, n_ctx=n_ctx, n_gpu_layers=n_gpu_layers) result = inst.chat(system_prompt, user_prompt, temperature=temperature, max_tokens=max_tokens) if auto_unload: inst.unload() return result def unload(): """Global unload.""" get_singleton().unload()