diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 02fa180..ebb2d2e 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -210,8 +210,9 @@ "templater-obsidian:Templater": false } }, - "active": "26813e16e829b798", + "active": "7066c9273c6fb936", "lastOpenFiles": [ + "100 inbox/Local LLM Infrastructure & Qwen2.5 Deployment.md", "010 inbox/Email & DNS Setup — Domains, Mailboxes & Records.md", "300 areas/360 Dev-Ops Network Computers/Email & DNS Setup — Domains, Mailboxes & Records.md", "000 daily/Daily Notes and To Do.md", @@ -239,7 +240,6 @@ "Clippings/alibabaopen-code-review Secure, fast, efficient, battle-tested at Alibaba's scale. Hybrid architecture code review tool deterministic pipelines + LLM Agent, precise line-level comments, built-in multi-language ruleset (NPE, thread-safety, XSS, S.md", "300 areas/395 AI Tools to try/Tools to try with AI.md", "200 projects/260 Ideas to Build Create Make/Artemis on GrapheneOS Pixel Mobile.md", - "400 Personal Family/460 Family Archive/Wilsons Prom Camp.md", "010 inbox/_WATCHER", "300 areas/355 AI Research and Techniques", "300 areas/355 Ai Research and Techniques", diff --git a/100 inbox/Local LLM Infrastructure & Qwen2.5 Deployment.md b/100 inbox/Local LLM Infrastructure & Qwen2.5 Deployment.md new file mode 100644 index 0000000..5cf5a56 --- /dev/null +++ b/100 inbox/Local LLM Infrastructure & Qwen2.5 Deployment.md @@ -0,0 +1,54 @@ +--- +created: 2026-10-12 14:07 +modified: 2026-10-12 14:07 +type: note +tags: [projects/local-llm, infra, ollama, qwen2.5] +aliases: [Local LLM Infrastructure, Qwen2.5 Deployment] +--- +# [[Local LLM Infrastructure & Qwen2.5 Deployment]] + +## 📌 Executive Summary +This project establishes an **on-demand local LLM inference node** on workstation **`sam-4screen-desktop` (`192.168.20.27`)**. It leverages the machine's 64 GB DDR4 RAM and 4-core Intel i7 CPU to run 14-billion parameter models locally (**Qwen2.5-Coder-14B** and **Qwen2.5-14B-Instruct**) without consuming idle background resources or requiring dedicated server GPUs. + +--- + +## 🖥️ System Architecture & Machine Roles + +* **Workstation `.27` (`sam-4screen-desktop`):** + * **Role:** Primary Local LLM Host. + * **Hardware:** Intel i7-7700K CPU + **64 GB DDR4 RAM** (~32 GB available for models). + * **Inference Strategy:** **Native CPU execution** using AVX2 instructions (bypasses Docker virtualization overhead for maximum CPU performance). + * **Auto-Unload:** Idle models automatically unload from RAM after 5 minutes (`OLLAMA_KEEP_ALIVE=5m`). + +* **Gateway Node `.13` (`nixos-desktop`):** + * **Role:** Network Gateway & OmniRoute Router. + * **Rule:** **Strictly no LLM inference runs on `.13`** due to memory constraints. + +--- + +## 🛠️ On-Demand Management Commands + +Simple terminal commands manage the Ollama service on `.27`: + +| Command | Action | Output / Behavior | +| :--- | :--- | :--- | +| `ai-on` | Starts local Ollama service | Binds to `0.0.0.0:11434` for local & LAN access. | +| `ai-off` | Stops local Ollama service | Immediately frees RAM and CPU cycles. | +| `ai-status` | Displays service status | Reports systemd unit state and port listeners. | + +--- + +## 🤖 Installed Local Models + +| Model | Size | Parameter Quantization | Primary Purpose | +| :--- | :--- | :--- | :--- | +| `qwen2.5-coder:14b` | 9.0 GB | Q4_K_M GGUF | Offline code generation, refactoring, and code review. | +| `qwen2.5:14b` | 9.0 GB | Q4_K_M GGUF | General drafting, reasoning, and context analysis. | + +--- + +## 💼 Skills & Technical Concepts Demonstrated + +* **Linux System Administration:** Systemd user unit management, environment variables, network binding (`0.0.0.0`), process lifecycle control. +* **Local AI & GGUF Inference:** CPU-optimized LLM deployment, RAM budgeting, model quantization trade-offs. +* **Edge & Hybrid Architecture:** On-demand local processing paired with cloud API fallback mechanisms.