diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 594530a..071db3f 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -27,7 +27,7 @@ "state": { "type": "markdown", "state": { - "file": "100 inbox/50 Legendary Websites.md", + "file": "010 inbox/50 Legendary Websites.md", "mode": "source", "source": false }, @@ -198,7 +198,7 @@ "state": { "type": "file-properties", "state": { - "file": "100 inbox/50 Legendary Websites.md" + "file": "010 inbox/50 Legendary Websites.md" }, "icon": "lucide-info", "title": "File properties" @@ -224,10 +224,10 @@ "templater-obsidian:Templater": false } }, - "active": "a912940b5902b787", + "active": "7066c9273c6fb936", "lastOpenFiles": [ "400 Personal Family/460 Family Archive/Harry Finn Trip Lorne Friends.md", - "100 inbox/50 Legendary Websites.md", + "010 inbox/50 Legendary Websites.md", "100 Table of Contents/Projects.md", "100 Table of Contents/Table of Contents.md", "100 Table of Contents/Tools & Software.md", @@ -252,7 +252,7 @@ "templates/Index.md", "templates/Container.md", "templates/Backup-System.md", - "100 inbox", + "010 inbox", "300 areas/350 AI/Tools to try", "400 Personal Family/490 Garden Food Houses Farms Hobbies", "400 Personal Family/480 Art Music Photography", diff --git a/000 daily/Daily Notes and To Do.md b/000 daily/Daily Notes and To Do.md deleted file mode 100644 index 8a05d94..0000000 --- a/000 daily/Daily Notes and To Do.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -id: 1778841999-CBOE -aliases: [] -tags: - - Obsidian - - Exrcise - - memory-vault - - Nvim ---- - -## TO DO - -- [x] Cook Chicken and Pasta βœ… 2026-08-10 -- [x] Wash dog sheets βœ… 2026-08-10 -- [x] Pack away electronics and record with homebox βœ… 2026-08-10 -- [x] Pack away snow gear and record with homebox βœ… 2026-08-10 -- [ ] Fix dualis lights -- [ ] Fix Prado lights -- [ ] Enter competition before August 31 https://www.seeedstudio.com/make-a-sign-interactive-signage-contest-2026 - -## 🎯 Focus Areas - -- [ ] Consolidate photos - ensure there is back up. Any photos added to photos such as from sd cards, temp transfer or missing ensure they are in immich -- [ ] Google CLI for photos to bring them down using command line gphotos-sync -- [ ] Document all work ready for Resume AI -- [ ] Gitea for old work on .13 machine with Caddy forwarding from .35 -- [ ] Remove duplicates, bad photos etc. -- [ ] Consolidate - - website work in htdocs - - Angry penguin - - Display Screen Media - - projects electronic, arduino - - websites - - movies - - music - - sheet music - - Notes on computers, family, ideas, TODO's - - -- [ ] Fix Audas Media NBN name to use it. Otherwise lose the domain. -- [ ] Consolidate all text and data for resume AI - projects, code, images etc. -- [ ] Move notes from latest work Pi Dev, Obsidian, OpenSpec, Archon, Plannotator, -- [ ] RustFS for links like s3 to container. -- [ ] Tuya local on home assistan -- [ ] Finish AI-resume set up. -- [ ] Set up Vorta back up with Borg -- [ ] Hermes desktop ? Alternatives? - - - - - -- [ ] Start mapping and logging projects, IOT, Garden to fill out the data for the AI Resume. - -- [ ] Caddy on .27 main dev machine for langfuse and other web interface things for resume. These will need to be moved for final production. - - -- Weekend - garden and gym. -- Finish logging trees with NFC. Use new WIKI ? Mulch front yard , move out trailers. -. diff --git a/100 inbox/50 Legendary Websites.md b/010 inbox/50 Legendary Websites.md similarity index 100% rename from 100 inbox/50 Legendary Websites.md rename to 010 inbox/50 Legendary Websites.md diff --git a/200 projects/210 AI Resume/TencentCloudTencentDB-Agent-Memory TencentDB Agent Memory is a team-level memory hub for AI Agents β€” turning conversations, docs, and code into four reusable memory assets (Chat Memory, Skill, LLM-Wiki, Code-Graph) that are governed, shared, and.md b/200 projects/210 AI Resume/TencentCloudTencentDB-Agent-Memory TencentDB Agent Memory is a team-level memory hub for AI Agents β€” turning conversations, docs, and code into four reusable memory assets (Chat Memory, Skill, LLM-Wiki, Code-Graph) that are governed, shared, and.md deleted file mode 100644 index 533e414..0000000 --- a/200 projects/210 AI Resume/TencentCloudTencentDB-Agent-Memory TencentDB Agent Memory is a team-level memory hub for AI Agents β€” turning conversations, docs, and code into four reusable memory assets (Chat Memory, Skill, LLM-Wiki, Code-Graph) that are governed, shared, and.md +++ /dev/null @@ -1,287 +0,0 @@ ---- -title: "TencentCloud/TencentDB-Agent-Memory: TencentDB Agent Memory is a team-level memory hub for AI Agents β€” turning conversations, docs, and code into four reusable memory assets (Chat Memory, Skill, LLM-Wiki, Code-Graph) that are governed, shared, and equipped across agents and frameworks." -source: "https://github.com/TencentCloud/TencentDB-Agent-Memory" -author: -published: -created: 2026-08-08 -description: "TencentDB Agent Memory is a team-level memory hub for AI Agents β€” turning conversations, docs, and code into four reusable memory assets (Chat Memory, Skill, LLM-Wiki, Code-Graph) that are governed, shared, and equipped across agents and frameworks. - TencentCloud/TencentDB-Agent-Memory" -tags: - - "clippings" ---- - -## Installation - -Start all three services in one go (`memory-core` + `memory-hub` + `proxy`): - -``` -git clone https://github.com/Tencent/TencentDB-Agent-Memory.git -cd TencentDB-Agent-Memory/deploy/global-images -cp .env.example .env -$EDITOR .env # Fill in two sets of LLM parameters (memory group + proxy group) -./start-all.sh # Launch everything with one command; when finished, it prints a one-liner you can paste directly into Claude -``` - -Open the panel: [http://localhost:8125](http://localhost:8125/). - -Complete installation documentation (standalone Memory Hub deployment, Proxy + Claude Code / CodeBuddy usage, stop and cleanup, port reference, etc.) is available in [**INSTALL.md**](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/INSTALL.md) (δΈ­ζ–‡: [INSTALL\_CN.md](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/INSTALL_CN.md)). - -### Migrating data from an older version - -If you're already on an older release (v1.x / v0.x) and want to bring your existing data over to v2.0.0+, we provide a migration tool: - -See [**Data Migration Tool (v2 β†’ v3)**](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/MemoryCore/scripts/migrate-v2-to-v3/README.md) for full usage and flags. New installations can skip this. - -## What is TencentDB Agent Memory? - -We started from a practical question: **How do you reduce repetitive work when using Agents?** - -If project context has already been explained, it shouldn't need to be repeated in a new session. If documents have already been read, every Agent shouldn't have to start again from page one. A workflow that already works shouldn't have to be rediscovered next time. - -Memory here means more than just "remembering conversations." **Any information that helps the next Agent avoid reinventing the wheel should be saved, organized, and reused.** - -``` -Existing information β†’ Reusable memory assets β†’ Fewer turns β†’ Less rework β†’ More stable results and higher efficiency -``` - -### Let experience accumulate, flow, and pass on to the next Agent - -**Memory Hub** for Agent teams closes the loop across the entire experience lifecycle: work produces assets, assets circulate through the team, and new members can load the team's save file on day one. - -1. **Automatic asset extraction**: Extract Chat Memory and Skills from conversations and tasks; convert documents and code into Wiki and CodeGraph; then manage, review, and route them consistently. -2. **Portable & multi-Agent compatible**: Memory assets are decoupled from Agent frameworks β€” they can move across frameworks and be shared and maintained by multiple Agents and team members. -3. **Cold-start friendly**: Import existing documents, codebases, and Agent conversation sessions. New Agent teams can start from existing experience instead of learning from scratch. - -### 🧠 A brain that remembers people and context - -- **Chat Memory** retains preferences, facts, decisions, and interaction history. -- Each Agent automatically gets its own memory when created β€” no need to re-introduce yourself next time. -- L0 Conversation β†’ L1 Atom β†’ L2 Scenario β†’ L3 Persona β€” raw conversations are distilled layer by layer. - -[![image.png](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/chat_memory.cn.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/chat_memory.cn.png) - -> "Don't refactor the old auth module β€” mobile is still using it." β€” Context this costly shouldn't depend on humans repeating it every time. - -### ⚑ A Skill library that accumulates expertise - -- After completing complex work, Agents can extract and manage reusable Skills from conversations and tool calls, and import them into the context of a designated Agent when needed. -- A Skill isn't just a prompt snippet; it has versions, resource files, trigger boundaries, execution steps, and validation rules. -- Personal Skills are private by default; after review, they can be shared with the team and assigned to other Agents. - -[![image.png](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/skill.cn.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/skill.cn.png) - -> Troubleshooting, code review, release checklists β€” learn it once, and the whole team can use it. - -### πŸ“– A knowledge map that reads both docs and code - -- **Wiki** turns product docs, design specs, and ops runbooks into structured pages with a link graph. (Inspired by Karpathy's LLM knowledge base.) - -[![image.png](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/wiki.cn.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/wiki.cn.png) - -- **CodeGraph** indexes code symbols, files, call relationships, and impact paths. - -[![image.png](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/codegraph.cn.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/codegraph.cn.png) - -- Agents can search, read, inspect callers/callees, and perform impact analysis before modifying code. - -> Wiki keeps Agents from reading every file list before getting to work. CodeGraph doesn't just tell them "the code is here" β€” it tells them "changing this might affect those." - -### πŸ›‘οΈ A team memory panel controlled by humans - -- Create teams and Agents in Memory Hub; review, share, and equip memory assets. -- Manage ownership, versions, status, visibility, usage counts, and Agent bindings in one place. -- `private` belongs strictly to the Owner; `team` is visible to all team members; `restricted` grants precise access via User / Role / Agent ACLs. -- Two role layers: **global System Admin** manages users and teams (creating teams, adding members) and can also use Wiki, CodeGraph, Skill, and other asset management features; **Team-level roles** include Admin (team manager) and Member (regular member), responsible for asset collaboration and access control within a team. Asset ownership is tracked via Owner β€” the Owner automatically has management permissions for their assets. - -[![image.png](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/asset.cn.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/asset.cn.png) - -## Cold Start: Load the Save File, Then Get to Work - -Most Agents' first task is re-learning your project. TencentDB Agent Memory turns the learning cost you've already paid into a save file: - -[![Cold Start: import codebase, docs, and history into Memory Hub](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/flowchart3.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/flowchart3.png) - -Specifically, these existing assets can be imported directly and processed automatically in the panel: - -- **Codebases**: Import existing repositories β€” **CodeGraph** automatically indexes symbols, files, call relationships, and impact paths. -- **Documents & files**: Import relevant docs and files β€” **Wiki** automatically generates structured pages with a link graph. -- **Conversation sessions**: Import past Agent conversation sessions β€” **Skills and Chat Memory** are automatically extracted as reusable assets. - -> Stop retraining every Agent. Give it the save file. - -## One Play Style: Build a Growing Agent Team for a One-Person Company - -Open Memory Hub and create a team: - -``` -Tiny but Serious Inc. -β”œβ”€β”€ πŸ‘€ You Β· Set goals / Make decisions -β”œβ”€β”€ πŸ”­ Scout Β· Research / Find opportunities -β”œβ”€β”€ πŸ›  Builder Β· Write code / Build products -β”œβ”€β”€ πŸ§ͺ Reviewer Β· Test / Find issues -└── 🧠 Agent Memory Β· Preserve the team's experience -``` - -You're not opening four disconnected chat windows β€” you're assembling a squad with different roles that can inherit the team's accumulated experience. - -### Recruit first, then equip - -``` -πŸ”­ Scout - β”œβ”€β”€ User interview Chat Memory - β”œβ”€β”€ Market research Wiki - └── Competitive analysis Skill - -πŸ›  Builder - β”œβ”€β”€ Product Wiki - β”œβ”€β”€ Project CodeGraph - └── Feature Delivery Skill - -πŸ§ͺ Reviewer - β”œβ”€β”€ Historical incident Chat Memory - β”œβ”€β”€ Project CodeGraph - └── Release Checklist Skill -``` - -Different roles, different loadouts. Less noise β€” give each Agent the memory assets it actually needs to get work done. - -**The company can be tiny. Experience can compound forever.** - -## Memory Assets, Not a Chat Log Warehouse - -RAG answers "what can be found?" Team Memory also answers "who can use it, which version is valid, and which Agent should receive it." - -| | Chat History | Standard RAG | TencentDB Agent Memory | -| ---------------------------------- | ------------ | ----------------- | ---------------------- | -| Cross-session user understanding | β–³ | β–³ | βœ… Chat Memory | -| Distilled executable experience | β€” | β€” | βœ… Skill | -| Document structure & relationships | β€” | β–³ Chunk retrieval | βœ… Wiki + Link Graph | -| Code call graphs & impact scope | β€” | β–³ Text match | βœ… CodeGraph | -| Ownership / Version / Status | β€” | β€” | βœ… | -| Team sharing & Agent loadout | β€” | β€” | βœ… | -| Private / Team / ACL | β€” | β–³ | βœ… | - -## Memory Hub Is Not a Display Board β€” It's a Control Panel - -| Play Style | What you do in the Hub | -| ---------------------- | -------------------------------------------------------------------------------- | -| **Team Up** | Create teams, add people and Agents, define sharing boundaries | -| **Asset Library** | Browse, search, review, and manage Chat Memory, Skills, Wiki, and CodeGraph | -| **Agent Loadout** | Bind different memory assets to different Agents; adjust priority and usage mode | -| **Knowledge Workshop** | Build Wiki and CodeGraph; monitor processing status and asset metadata | -| **Access Control** | Switch between private, team, and ACL-based access; revoke sharing when needed | - -When you open an asset, what matters is not just "what it says," but also "where it came from, which version it is, who it's assigned to, and whether it's been used recently." - -## Every Loop Gains Experience - -[![Every Loop Gains Experience: continuous accumulation, making every use smarter](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/flowchart4.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/flowchart4.png) - -Memory doesn't run the Agent loop; it ensures the next iteration inherits the previous one's results: valuable interactions stay in Chat Memory, proven workflows are distilled into Skills, and document/code changes are updated through Wiki ingest and CodeGraph sync. - -**Without Memory, loops may just repeat faster. With inherited memory, each iteration has the chance to be better than the last.** - -New Chat Memory and Skills are private by default. Sharing is an explicit action, not a default leak. - -| Visibility | Semantics | -| ------------ | ----------------------------------------------------- | -| `private` | Only the Owner can read β€” not even team admins | -| `team` | Team members can read; the Owner / Admin can manage | -| `restricted` | Precise access via User / Role / Agent ACL | -| `agent` | For targeted equipping of Agents within the same team | - -You can assign the "Release Skill" to the Release Agent, the "Architecture Wiki" to all development Agents, and CodeGraph to Coder and Reviewer. - -## Technical Implementation - -TencentDB Agent Memory doesn't aim to "store everything." It solves three problems: **what's worth keeping, who can use it, and how to retrieve less while retrieving the right things next time.** - -[![Technical overview: layering (L0–L3), Memory Assets, Memory Hub, identity-based assembly for Agents](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/flowchart5.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/flowchart5.png) - -### 1\. Memory isn't flat records β€” it grows in layers - -Conversations are first saved as L0, then refined by an async pipeline into multiple levels of granularity: - -| Layer | What it stores | Primary use | -| --- | --- | --- | -| **L0 Conversation** | Raw conversations with full context | Verify exact wording, timestamps, and sources | -| **L1 Atom** | Facts, preferences, constraints, and events extracted from conversations | Precise recall of actionable information | -| **L2 Scenario** | Knowledge blocks organized around projects or scenarios | Quickly restore a working context | -| **L3 Core / Persona** | Long-term profiles, stable patterns, and high-level cognition | Let Agents rapidly enter a user's and team's context | - -Both generation and retrieval are layered: normally, L2/L3 provide a quick context bootstrap; when specific facts are needed, BM25 + vector retrieval + RRF fall back to L1/L0. Results are further capped by item count, character budget, and timeout limits to prevent memory from overwhelming the context window. - -### 2\. Memory isn't a global prompt β€” it's the Agent's loadout - -Chat Memory, Skills, Wiki, and CodeGraph are all registered uniformly as Memory Assets. Memory Hub uses **Fixed Binding + ACL** to determine which assets a given Agent can use: first narrow the permission scope by Team, User, Agent, and visibility, then retrieve based on the current query. - -This lets teams share experience without exposing all their private information; switching Agents or frameworks only requires re-equipping, not retraining. - -### 3\. Knowledge isn't injected wholesale β€” it's called on demand - -Documents are organized into searchable Wiki pages that support link-graph drill-down; codebases are indexed into CodeGraph assets containing files, symbols, and call relationships. Agents first discover capabilities via `/v3/tools/list`, then use `/v3/tools/call` to read relevant pages, source code, or impact paths. - -This makes documents and code part of memory as well β€” but they remain available tools that only enter context when truly needed. - -## Benchmark - -| Benchmark | Without TencentDB Agent Memory | With it enabled | Relative improvement | -| --- | --- | --- | --- | -| **PersonaMem** | 48% | **76%** | **+59%** | - -PersonaMem tests whether an Agent can correctly understand and apply user information after extended interactions. - -## Notes - -- Wiki and CodeGraph are built asynchronously; allow some processing time before they reach `ready` status. -- CodeGraph currently prioritizes public HTTPS repositories; support for private repositories and SSH credentials is still being refined. -- The Hub supports manual asset binding; fully automated memory routing is still under iteration. -- TencentDB Agent Memory currently supports OpenClaw, Hermes, Claude Code, CodeBuddy, and SDK integration; broader cross-framework migration is on the roadmap. - -## Related Documentation - -- [Full Installation Guide](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/INSTALL.md) (Memory Core + Hub + Proxy one-click deployment) -- [Data Migration Tool (v2 β†’ v3)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/MemoryCore/scripts/migrate-v2-to-v3/README.md) (if you're on an older release and want to migrate existing data) -- [Knowledge OpenAPI](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/MemoryKnowledge/openapi.yaml) -- [Contributing Guide](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/CONTRIBUTING.md) - -Agent Memory doesn't have a settled standard yet. Bug reports, documentation, benchmarks, new framework adapters, and more creative Memory Hub use cases are all welcome. - ---- - -## Acknowledgements - -TencentDB Agent Memory stands on the shoulders of the open-source community: - -- [**CodeGraph**](https://github.com/colbymchenry/codegraph) β€” our CodeGraph asset module **uses code from this project**. Its design of a pre-indexed code graph is the foundation of our implementation. -- [**Hermes Agent**](https://github.com/nousresearch/hermes-agent) (Nous Research) β€” our Skill asset management **uses part of the Skill-related code from Hermes Agent and builds further optimizations base on it**. -- [**"LLM Wiki"** by Andrej Karpathy](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) β€” the idea of treating documentation as an LLM-maintained, incrementally growing knowledge artifact directly informed how our Wiki layer is built and kept up to date. - -We are grateful to the authors and contributors of these projects. - ---- - -## Community & Contributing - -We welcome contributions of all kinds β€” bug reports, feature suggestions, documentation fixes, benchmark reproductions, ecosystem integrations, or pull requests. Agent memory is far from settled, and we hope to build it together with the community. - -- 🐞 **Found a bug or have a question?** Open an issue in [GitHub Issues](https://github.com/Tencent/TencentDB-Agent-Memory/issues) β€” we respond within 24 hours. -- πŸ’‘ **Have an idea to share?** Start a thread in [GitHub Discussions](https://github.com/Tencent/TencentDB-Agent-Memory/discussions). -- πŸ› οΈ **Want to contribute code?** Please read [CONTRIBUTING.md](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/CONTRIBUTING.md) first. -- πŸ’¬ **Want to chat with us?** Join our [Discord community](https://discord.gg/dJQM6mKMF) and talk to the core developers directly. - ---- - -Let the path the team has walked become the next Agent's starting line. - ---- - -## ✨ Contributors - -> πŸ’‘ Thanks to the following contributors building with us β€” you make TencentDB Agent Memory better. - -[![](https://camo.githubusercontent.com/9447bd6c00d7b3ef65d5f15ae3eaab34cfcd6fa004f69a008dcd5531bfa280d5/68747470733a2f2f636f6e747269622e726f636b732f696d6167653f7265706f3d54656e63656e74436c6f75642f54656e63656e7444422d4167656e742d4d656d6f727926636f6c756d6e733d313226616e6f6e3d31)](https://github.com/TencentCloud/TencentDB-Agent-Memory/graphs/contributors) - -| **If TencentDB Agent Memory has been helpful to you, please consider starring the project.** If you have any suggestions, feel free to open an issue for discussion. | [![Star TencentDB Agent Memory](https://github.com/TencentCloud/TencentDB-Agent-Memory/raw/feat/server_team/assets/images/star-helper.png)](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/assets/images/star-helper.png) | -| --- | --- | - -[MIT](https://github.com/TencentCloud/TencentDB-Agent-Memory/blob/feat/server_team/LICENSE) Β© TencentDB Agent Memory Team \ No newline at end of file diff --git a/300 areas/360 Dev-Ops Network Computers/Websites on Nixos-Dekstop 13.md b/300 areas/360 Dev-Ops Network Computers/Websites on Nixos-Dekstop 13.md deleted file mode 100644 index 744f5bd..0000000 --- a/300 areas/360 Dev-Ops Network Computers/Websites on Nixos-Dekstop 13.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -created: 2026-08-05 13:11 -modified: 2026-08-26 12:20 -type: note -tags: - - websites - - family-home-lab - - portfolio -aliases: - - Websites ---- -# Websites on Nixos-Desktop 13 - -Websites / services I've built and deployed for the **Family Home Lab**. Public -sites live under `*.lab.audasmedia.com.au` (reachable off-LAN); `*.home.lab` are -LAN-only. Source of truth: this repo (`sam/family_home_lab` on Gitea) β†’ -`docs/websites.md`. - -## Core β€” the portal & chat -- **console.lab.audasmedia.com.au** β€” the Family Home Lab console / home-dash - (FastAPI + Jinja + HTMX). Lists every service, has an Admin area (users, - passwords, Pi-dashboard sessions). -- **dsh-sam / dsh-jo / dsh-harry / dsh-finn.lab** β€” DeepSeek Harness chat - (one per family member). Memory sidebar, web/doc summarise, image ingest + - generation. Hardened (no shell). -- **chat.lab** (family group chat) + **pb.chat.lab** (admin). - -## Media tools -- **gimp.lab** β€” GIMP (image editor) *(Photopea removed β€” image had no app assets)* -- **video.lab** β€” KdenLive (video editor) -- **audio.lab** β€” Audacity (audio editor) -- **lmms.lab** β€” LMMS music studio - -## Resume / portfolio -- **sam-developer.lab**, **sam-devops.lab**, **sam-iot-electronics.lab**, - **sam-pursuits.lab** β€” resume sites -- **wikijs.lab** β€” resume wiki -- **lynx.lab**, **sequence.lab** - -## Supporting / infra -- **omniroute.lab** β€” AI router dashboard -- **photo-filter.lab** -- **gitea.lab** β€” Git hosting (`sam/family_home_lab`, `sam/pi-config`, …) - -## How deploy works -- Portal + dsh live on `.13` (Docker). Caddy reverse proxy on `.35`. -- Code is in this repo and pushed to **Gitea**: `sam/family_home_lab` (SSH - `ssh://git@gitea.lab.audasmedia.com.au:2222/sam/family_home_lab.git`). - -*Full inventory: [[Home Network Map Overview]] and repo `docs/websites.md`.* \ No newline at end of file diff --git a/300 areas/395 AI Tools to try/D4VinciScrapling πŸ•·οΈ An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here httpsdiscord.ggEMgGbDceNQ.md b/300 areas/395 AI Tools to try/D4VinciScrapling πŸ•·οΈ An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here httpsdiscord.ggEMgGbDceNQ.md deleted file mode 100644 index 4dec810..0000000 --- a/300 areas/395 AI Tools to try/D4VinciScrapling πŸ•·οΈ An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here httpsdiscord.ggEMgGbDceNQ.md +++ /dev/null @@ -1,464 +0,0 @@ ---- -title: "D4Vinci/Scrapling: πŸ•·οΈ An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ" -source: "https://github.com/D4Vinci/Scrapling" -author: -published: -created: 2026-09-15 -description: "πŸ•·οΈ An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ - D4Vinci/Scrapling" -tags: - - "clippings" ---- -## Effortless Web Scraping for the Modern Web - -[![D4Vinci%2FScrapling | Trendshift](https://camo.githubusercontent.com/835790e5d69ed074841a94212e23721278ed5831280d57e2b1dd0622cb67dd80/68747470733a2f2f7472656e6473686966742e696f2f6170692f62616467652f7265706f7369746f726965732f3134323434)](https://trendshift.io/repositories/14244) - -[**Selection methods**](https://scrapling.readthedocs.io/en/latest/parsing/selection.html) Β· [**Fetchers**](https://scrapling.readthedocs.io/en/latest/fetching/choosing.html) Β· [**Spiders**](https://scrapling.readthedocs.io/en/latest/spiders/architecture.html) Β· [**Proxy Rotation**](https://scrapling.readthedocs.io/en/latest/spiders/proxy-blocking.html) Β· [**CLI**](https://scrapling.readthedocs.io/en/latest/cli/overview.html) Β· [**MCP**](https://scrapling.readthedocs.io/en/latest/ai/mcp-server.html) - -Scrapling is an adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl. - -Its parser learns from website changes and automatically relocates your elements when pages update. Its fetchers bypass anti-bot systems like Cloudflare Turnstile out of the box. And its spider framework lets you scale up to concurrent, multi-session crawls with pause/resume, automatic proxy rotation, and a crawl speed that adapts to how fast each website responds and backs off when it starts blocking you - all in a few lines of Python. One library, zero compromises. - -Blazing fast crawls with real-time stats and streaming. Built by Web Scrapers for Web Scrapers and regular users, there's something for everyone. - -``` -from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher -StealthyFetcher.adaptive = True -p = StealthyFetcher.fetch('https://example.com', headless=True, network_idle=True) # Fetch website under the radar! -products = p.css('.product', auto_save=True) # Scrape data that survives website design changes! -products = p.css('.product', adaptive=True) # Later, if the website structure changes, pass \`adaptive=True\` to find them! -``` - -Or scale up to full crawls - -``` -from scrapling.spiders import Spider, Response - -class MySpider(Spider): - name = "demo" - start_urls = ["https://example.com/"] - - async def parse(self, response: Response): - for item in response.css('.product'): - yield {"title": item.css('h2::text').get()} - -MySpider().start() -``` - -[![At DataImpulse, we specialize in developing custom proxy services for your business. Make requests from anywhere, collect data, and enjoy fast connections with our premium proxies.](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/DataImpulse.png)](https://dataimpulse.com/?utm_source=scrapling&utm_medium=banner&utm_campaign=scrapling) - -## Platinum Sponsors - -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/coldproxy.png)](https://coldproxy.com/?utm_source=scrapling&utm_medium=sponsorship&utm_campaign=202609-scrapling-readme&utm_content=platinum-badge&promo=SCRAPLING5 "Residential, IPv6 & Datacenter Proxies for Web Scraping") | [**ColdProxy**](https://coldproxy.com/?utm_source=scrapling&utm_medium=sponsorship&utm_campaign=202609-scrapling-readme&utm_content=platinum-badge&promo=SCRAPLING5) provides residential and datacenter proxies for stable web scraping, public data collection, and geo-targeted testing across 195+ countries. *Using this link gives you 5% off*. | -| --- | --- | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/NodeMaven.jpg)](https://go.nodemaven.com/scraplingaugust "Proxies with the Highest IP Scores") | [NodeMaven](https://go.nodemaven.com/scraplingaugust) - The most efficient proxy provider for Web Scraping and Automation with the Highest Quality IP on the market. Use code SCRAPLING35 for 35% discount. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/proxidize.png)](https://proxidize.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=scrapling&utm_content=d4vinci "Clean Proxies with No Nonsense.") | [**Proxidize**](https://proxidize.com/?utm_source=github&utm_medium=sponsorship&utm_campaign=scrapling&utm_content=d4vinci) provides mobile and residential proxies for scraping, browser automation, SEO monitoring, AI agents, and data collection. *Use code **scrapling20** for 20% off*. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/HyperSolutions.png)](https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling "Bot Protection Bypass API for Akamai, DataDome, Incapsula & Kasada") | Scrapling handles Cloudflare Turnstile. For enterprise-grade protection, [**Hyper Solutions**](https://hypersolutions.co/?utm_source=github&utm_medium=readme&utm_campaign=scrapling) provides API endpoints that generate valid antibot tokens for **Akamai**, **DataDome**, **Kasada**, and **Incapsula**. Simple API calls, no browser automation required. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/BirdProxies.jpg)](https://birdproxies.com/t/scrapling "At Bird Proxies, we eliminate your pains such as banned IPs, geo restriction, and high costs so you can focus on your work.") | Hey, we built [**BirdProxies**](https://birdproxies.com/t/scrapling) because proxies shouldn't be complicated or overpriced. Fast residential and ISP proxies in 195+ locations, fair pricing, and real support. **Try our FlappyBird game on the landing page for free data!** | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png)](https://evomi.com/?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling "Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB") | [**Evomi**](https://evomi.com/?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling) : residential proxies from $0.49/GB. Scraping browser with fully spoofed Chromium, residential IPs, auto CAPTCHA solving, and anti-bot bypass. **Scraper API for hassle-free results. MCP and N8N integrations are available.** | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/TikHub.jpg)](https://tikhub.io/?utm_source=github.com/D4Vinci/Scrapling&utm_medium=marketing_social&utm_campaign=retargeting&utm_content=carousel_ad "Unlock the Power of Social Media Data & AI") | [TikHub.io](https://tikhub.io/?utm_source=github.com/D4Vinci/Scrapling&utm_medium=marketing_social&utm_campaign=retargeting&utm_content=carousel_ad) provides 900+ stable APIs across 16+ platforms including TikTok, X, YouTube & Instagram, with 40M+ datasets. Also offers [DISCOUNTED AI models](https://ai.tikhub.io/?ref=KarimShoair) - Claude, GPT, GEMINI & more up to 71% off. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png)](https://petrosky.io/d4vinci "PetroSky delivers cutting-edge VPS hosting.") | Close your laptop. Your scrapers keep running. [PetroSky VPS](https://petrosky.io/d4vinci) - cloud servers built for nonstop automation. Windows and Linux machines with full control. From €6.99/mo. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/TWSC.png)](https://substack.thewebscraping.club/p/scrapling-hands-on-guide?utm_source=github&utm_medium=repo&utm_campaign=scrapling "The #1 newsletter dedicated to Web Scraping") | Read a full review of [Scrapling on The Web Scraping Club](https://substack.thewebscraping.club/p/scrapling-hands-on-guide?utm_source=github&utm_medium=repo&utm_campaign=scrapling) (Nov 2025), the #1 newsletter dedicated to Web Scraping. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SwiftProxy.png)](https://www.swiftproxy.net/?ref=D4Vinci "Scalable Solutions for Web Data Access") | [Swiftproxy](https://www.swiftproxy.net/?ref=D4Vinci) provides scalable residential proxies with 80M+ IPs across 195+ countries, delivering fast, reliable connections, automatic rotation, and strong anti-block performance. Free trial available. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/niuproxy.png)](https://niuproxy.com/?utm_source=scrapling&utm_medium=scrapling&ref=scrapling "Affordable Residential in 190+ Countries") | [NiuProxy](https://niuproxy.com/?utm_source=scrapling&utm_medium=scrapling&ref=scrapling) β€” Rotating residential proxies from $0.35/GB. Use exclusive Scrapling code PAY2 for 10% off your recharge. | -| [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/voidmob.png)](https://voidmob.com/proxies?utm_source=scrapling "Premium Mobile Proxies from Real Carrier Networks") | [VoidMob](https://voidmob.com/proxies?utm_source=scrapling): 4G/5G mobile proxies on real SIMs and devices. Dedicated or shared, sticky, rotating, p0f fingerprint presets, API/MCP and full protocol support. SCRAPLING20 for 20% off. | - -*Do you want to show your ad here? Click here* - -## Sponsors - -[![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/novada.jpg)](https://www.novada.com/?d4vinci-scrapling "The All-in-One Solution for Every Data Scraping Scenario") [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/cloro.jpg)](https://cloro.dev/?utm_source=referral&utm_medium=scrapling "The search API for the AI era") - -[![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png)](https://serpapi.com/?utm_source=scrapling "Scrape Google and other search engines with SerpApi") [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/decodo.png)](https://visit.decodo.com/Dy6W0b "Try the Most Efficient Residential Proxies for Free") [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/ProxyEmpire.png)](https://proxyempire.io/?ref=scrapling&utm_source=github&utm_medium=paid_referral&utm_campaign=open_source_sponsorship&utm_content=scrapling "Collect The Data Your Project Needs with the Best Residential Proxies") [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/webshare.png)](https://www.webshare.io/?referral_code=48r2m2cd5uz1 "The Most Reliable Proxy with Unparalleled Performance") [![](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/proxiware.png)](https://proxiware.com/?ref=scrapling "Collect Any Data. At Any Scale.") - -*Do you want to show your ad here? Click here and choose the tier that suits you!* - ---- - -## Key Features - -### Spiders - A Full Crawling Framework - -- πŸ•·οΈ **Scrapy-like Spider API**: Define spiders with `start_urls`, async `parse` callbacks, and `Request` / `Response` objects. -- ⚑ **Concurrent Crawling**: Configurable concurrency limits, per-domain throttling, and download delays. -- πŸ”„ **Multi-Session Support**: Unified interface for HTTP requests, and stealthy headless browsers in a single spider - route requests to different sessions by ID. -- πŸ’Ύ **Pause & Resume**: Checkpoint-based crawl persistence. Press Ctrl+C for a graceful shutdown; restart to resume from where you left off. -- πŸ“‘ **Streaming Mode**: Stream scraped items as they arrive via `async for item in spider.stream()` with real-time stats - ideal for UI, pipelines, and long-running crawls. -- πŸ›‘οΈ **Blocked Request Detection**: Automatic detection and retry of blocked requests with customizable logic. -- 🚦 **AutoThrottle**: Stop guessing delays. The spider tunes the delay of each domain on its own from how fast the website responds, then doubles it (or waits what `Retry-After` asks) whenever the website starts blocking or rate-limiting you, and speeds back up once it stops. -- πŸ€– **Robots.txt Compliance**: Optional `robots_txt_obey` flag that respects `Disallow`, `Crawl-delay`, and `Request-rate` directives with per-domain caching. -- πŸ§ͺ **Development Mode**: Cache responses to disk on the first run and replay them on subsequent runs - iterate on your `parse()` logic without re-hitting the target servers. -- 🧩 **Ready-made Spider Templates**: Skip the boilerplate with `CrawlSpider` for rule-based link following, `SitemapSpider` for sitemap/robots.txt-driven crawls, `XMLFeedSpider` / `CSVFeedSpider` for iterating XML/RSS and CSV feeds, and `ShopifySpider` to pull every product out of any Shopify store through its JSON API, one item per variant. -- πŸ”— **Link Extraction**: A standalone `LinkExtractor` primitive with allow/deny patterns, domain filters, CSS/XPath scoping, extension filtering, and canonicalization - use it inside the templates or on its own. -- πŸ“¦ **Built-in Export**: Export results through hooks and your own pipeline or the built-in JSON/JSONL/CSV/XML exporters with `result.items.to_json()`, `to_jsonl()`, `to_csv()`, and `to_xml()`. - -### Advanced Websites Fetching with Session Support - -- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class. Can impersonate browsers' TLS fingerprint, headers, and use HTTP/3. -- **Dynamic Loading**: Fetch dynamic websites with full browser automation through the `DynamicFetcher` class supporting Playwright's Chromium and Google's Chrome. -- **Anti-bot Bypass**: Advanced stealth capabilities with `StealthyFetcher` and fingerprint spoofing. Can easily bypass all types of Cloudflare's Turnstile/Interstitial with automation. -- **Session Management**: Persistent session support with `FetcherSession`, `StealthySession`, and `DynamicSession` classes for cookie and state management across requests. -- **Proxy Rotation**: Built-in `ProxyRotator` with cyclic or custom rotation strategies across all session types, plus per-request proxy overrides. -- **Domain & Ad Blocking**: Block requests to specific domains (and their subdomains) or enable built-in ad blocking (~3,500 known ad/tracker domains) in browser-based fetchers. -- **DNS Leak Prevention**: Optional DNS-over-HTTPS support to route DNS queries through Cloudflare's DoH, preventing DNS leaks when using proxies. -- **Remote Browsers**: Instead of launching a browser locally, connect to one that's already running through CDP with `cdp_url`, whether it's on the same machine, another host, or a managed browser provider. You can also point any browser fetcher at your own Chromium build with `executable_path`. -- **Background API Capture**: Pass a URL pattern to `capture_xhr`, and all matching XHR/fetch responses the page makes while loading are collected for you as `Response` objects in `response.captured_xhr` - grab a site's API data without reverse-engineering the requests yourself. -- **Async Support**: Complete async support across all fetchers and dedicated async session classes. - -### Adaptive Scraping - -- πŸ”„ **Smart Element Tracking**: Relocate elements after website changes using intelligent similarity algorithms. -- 🎯 **Smart Flexible Selection**: CSS selectors, XPath selectors, filter-based search, text search, regex search, and more. -- πŸ” **Find Similar Elements**: Automatically locate elements similar to found elements. - -### AI Features - -- πŸ€– **MCP Server**: Let AI chatbots and agents (Claude/Cursor/etc) scrape through Scrapling with one-shot or session-based tools covering plain HTTP requests (any method), browser fetches, and stealth fetches that bypass Cloudflare. Pages are narrowed with CSS selectors and stripped of prompt-injection content before the AI sees them, so the agent reads less, costs less, and can't be hijacked by hidden text. Screenshots, remote browsers over CDP, and a secure-by-default HTTP transport are included. ([demo video](https://www.youtube.com/watch?v=qyFk3ZNwOxE)) -- 🧠 **Agent Skill**: A ready-to-install [Agent Skill](https://scrapling.readthedocs.io/en/latest/ai/agent-skill.html) that teaches coding agents the whole library, so the code they write with Scrapling matches the current API instead of guessing. -- πŸ“š **RAG-ready Markdown**: Turn any page into clean, sanitized, LLM-ready Markdown with one line (`page.markdown()`), or crawl a whole website into a Markdown corpus with the `SiteToMarkdownSpider` template, all without an LLM in the loop. ([docs](https://scrapling.readthedocs.io/en/latest/ai/building-rag-systems.html)) - -### High-Performance & battle-tested Architecture - -- πŸš€ **Lightning Fast**: Optimized performance outperforming most Python scraping libraries. -- πŸ”‹ **Memory Efficient**: Optimized data structures and lazy loading for a minimal memory footprint. -- ⚑ **Fast JSON Serialization**: 10x faster than the standard library. -- πŸ—οΈ **Battle tested**: Not only does Scrapling have 92% test coverage and full type hints coverage, but it has been used daily by hundreds of Web Scrapers over the past year. - -### Developer/Web Scraper Friendly Experience - -- 🎯 **Interactive Web Scraping Shell**: Optional built-in IPython shell with Scrapling integration, shortcuts, and new tools to speed up Web Scraping scripts development, like converting curl requests to Scrapling requests and viewing requests results in your browser. -- πŸš€ **Use it directly from the Terminal**: Optionally, you can use Scrapling to scrape a URL without writing a single line of code! -- πŸ› οΈ **Rich Navigation API**: Advanced DOM traversal with parent, sibling, and child navigation methods. -- 🧬 **Enhanced Text Processing**: Built-in regex, cleaning methods, and optimized string operations. -- πŸ“ **Auto Selector Generation**: Generate robust CSS/XPath selectors for any element. -- πŸ”Œ **Familiar API**: Similar to Scrapy/BeautifulSoup with the same pseudo-elements used in Scrapy/Parsel. -- 🀝 **Drop-in Scrapy Integration**: Already invested in Scrapy? Decorate any callback with `scrapling_response` to parse the responses you already fetch with Scrapling's parser, no rewrite needed. -- πŸ“˜ **Complete Type Coverage**: Full type hints for excellent IDE support and code completion. The entire codebase is automatically scanned with **PyRight** and **MyPy** with each change. -- πŸ”‹ **Ready Docker image**: With each release, a Docker image containing all browsers is automatically built and pushed. - -## Getting Started - -Let's give you a quick glimpse of what Scrapling can do without deep diving. - -### Basic Usage - -HTTP requests with session support - -``` -from scrapling.fetchers import Fetcher, FetcherSession - -with FetcherSession(impersonate='chrome') as session: # Use latest version of Chrome's TLS fingerprint - page = session.get('https://quotes.toscrape.com/', stealthy_headers=True) - quotes = page.css('.quote .text::text').getall() - -# Or use one-off requests -page = Fetcher.get('https://quotes.toscrape.com/') -quotes = page.css('.quote .text::text').getall() -``` - -Advanced stealth mode - -``` -from scrapling.fetchers import StealthyFetcher, StealthySession - -with StealthySession(headless=True, solve_cloudflare=True) as session: # Keep the browser open until you finish - page = session.fetch('https://nopecha.com/demo/cloudflare', google_search=False) - data = page.css('#padded_content a').getall() - -# Or use one-off request style, it opens the browser for this request, then closes it after finishing -page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare') -data = page.css('#padded_content a').getall() -``` - -Full browser automation - -``` -from scrapling.fetchers import DynamicFetcher, DynamicSession - -with DynamicSession(headless=True, disable_resources=False, network_idle=True) as session: # Keep the browser open until you finish - page = session.fetch('https://quotes.toscrape.com/', load_dom=False) - data = page.xpath('//span[@class="text"]/text()').getall() # XPath selector if you prefer it - -# Or use one-off request style, it opens the browser for this request, then closes it after finishing -page = DynamicFetcher.fetch('https://quotes.toscrape.com/') -data = page.css('.quote .text::text').getall() -``` - -### Spiders - -Build full crawlers with concurrent requests, multiple session types, and pause/resume: - -``` -from scrapling.spiders import Spider, Request, Response - -class QuotesSpider(Spider): - name = "quotes" - start_urls = ["https://quotes.toscrape.com/"] - concurrent_requests = 10 - - async def parse(self, response: Response): - for quote in response.css('.quote'): - yield { - "text": quote.css('.text::text').get(), - "author": quote.css('.author::text').get(), - } - - next_page = response.css('.next a') - if next_page: - yield response.(next_page[0].attrib['href']) - -result = QuotesSpider().start() -print(f"Scraped {len(result.items)} quotes") -result.items.to_json("quotes.json") -``` - -Use multiple session types in a single spider: - -``` -from scrapling.spiders import Spider, Request, Response -from scrapling.fetchers import FetcherSession, AsyncStealthySession - -class MultiSessionSpider(Spider): - name = "multi" - start_urls = ["https://example.com/"] - - def configure_sessions(self, manager): - manager.add("fast", FetcherSession(impersonate="chrome")) - manager.add("stealth", AsyncStealthySession(headless=True), lazy=True) - - async def parse(self, response: Response): - for link in response.css('a::attr(href)').getall(): - # Route protected pages through the stealth session - if "protected" in link: - yield Request(link, sid="stealth") - else: - yield Request(link, sid="fast", callback=self.parse) # explicit callback -``` - -Pause and resume long crawls with checkpoints by running the spider like this: - -``` -QuotesSpider(crawldir="./crawl_data").start() -``` - -Press Ctrl+C to pause gracefully - progress is saved automatically. Later, when you start the spider again, pass the same `crawldir`, and it will resume from where it stopped. - -Or skip writing the crawling logic altogether with the ready-made templates, like pulling an entire Shopify store's catalog: - -``` -from scrapling.spiders import ShopifySpider - -class MyStore(ShopifySpider): - target_website = "example.com" - -result = MyStore().start() # Every product in the store, one item per variant -``` -``` -from scrapling.fetchers import Fetcher - -# Rich element selection and navigation -page = Fetcher.get('https://quotes.toscrape.com/') - -# Get quotes with multiple selection methods -quotes = page.css('.quote') # CSS selector -quotes = page.xpath('//div[@class="quote"]') # XPath -quotes = page.find_all('div', {'class': 'quote'}) # BeautifulSoup-style -# Same as -quotes = page.find_all('div', class_='quote') -quotes = page.find_all(['div'], class_='quote') -quotes = page.find_all(class_='quote') # and so on... -# Find element by text content -quotes = page.find_by_text('quote', tag='div') - -# Advanced navigation -quote_text = page.css('.quote')[0].css('.text::text').get() -quote_text = page.css('.quote').css('.text::text').getall() # Chained selectors -first_quote = page.css('.quote')[0] - = first_quote.next_sibling.css('.author::text') -parent_container = first_quote.parent - -# Element relationships and similarity -similar_elements = first_quote.find_similar() -below_elements = first_quote.below_elements() -``` - -You can use the parser right away if you don't want to fetch websites like below: - -``` -from scrapling.parser import Selector - -page = Selector("...") -``` - -And it works precisely the same way! - -### Async Session Management Examples - -``` -import asyncio -from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession - -async with FetcherSession(http3=True) as session: # \`FetcherSession\` is context-aware and can work in both sync/async patterns - page1 = session.get('https://quotes.toscrape.com/') - page2 = session.get('https://quotes.toscrape.com/', impersonate='firefox135') - -# Async session usage -async with AsyncStealthySession(max_pages=2) as session: - tasks = [] - urls = ['https://example.com/page1', 'https://example.com/page2'] - - for url in urls: - task = session.fetch(url) - tasks.append(task) - - print(session.get_pool_stats()) # Optional - The status of the browser tabs pool (busy/free/error) - results = await asyncio.gather(*tasks) - print(session.get_pool_stats()) -``` - -## CLI & Interactive Shell - -Scrapling includes a powerful command-line interface: - -Launch the interactive Web Scraping shell - -``` -scrapling shell -``` - -Extract pages to a file directly without programming (Extracts the content inside the `body` tag by default). If the output file ends with `.txt`, then the text content of the target will be extracted. If it ends in `.md`, it will be a Markdown representation of the HTML content; if it ends in `.html`, it will be the HTML content itself. - -``` -scrapling extract get 'https://example.com' content.md -scrapling extract get 'https://example.com' content.txt --css-selector '#fromSkipToProducts' --impersonate 'chrome' # All elements matching the CSS selector '#fromSkipToProducts' -scrapling extract fetch 'https://example.com' content.md --css-selector '#fromSkipToProducts' --no-headless -scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' captchas.html --css-selector '#padded_content a' --solve-cloudflare -``` - -> [!note] Note -> There are many additional features, but we want to keep this page concise, including the MCP server and the interactive Web Scraping Shell. Check out the full documentation [here](https://scrapling.readthedocs.io/en/latest/) - -## Performance Benchmarks - -Scrapling isn't just powerful-it's also blazing fast. The following benchmarks compare Scrapling's parser with the latest versions of other popular libraries. - -### Text Extraction Speed Test (5000 nested elements) - -| # | Library | Time (ms) | vs Scrapling | -| --- | --- | --- | --- | -| 1 | Scrapling | 1.99 | 1.0x | -| 2 | Parsel/Scrapy | 2.06 | 1.035 | -| 3 | Raw Lxml | 2.56 | 1.286 | -| 4 | PyQuery | 23.98 | ~12x | -| 5 | Selectolax | 197.02 | ~99x | -| 6 | MechanicalSoup | 1545.15 | ~776.5x | -| 7 | BS4 with Lxml | 1562.1 | ~785.0x | -| 8 | BS4 with html5lib | 3412.73 | ~1714.9x | - -### Element Similarity & Text Search Performance - -Scrapling's adaptive element finding capabilities significantly outperform alternatives: - -| Library | Time (ms) | vs Scrapling | -| --- | --- | --- | -| Scrapling | 2.3 | 1.0x | -| AutoScraper | 12.58 | 5.47x | - -> All benchmarks represent averages of 100+ runs. See [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology. - -## Installation - -Scrapling requires Python 3.10 or higher: - -``` -pip install scrapling -``` - -> [!important] Important -> This installation only includes the parser engine and its dependencies, without any fetchers or commandline dependencies. So importing anything from `scrapling.fetchers` or `scrapling.spiders`, like in the examples above, will raise `ModuleNotFoundError` with this installation alone. If you are going to use any of the fetchers or spiders, install the fetchers' dependencies first as shown below. - -### Optional Dependencies - -1. If you are going to use any of the extra features below, the fetchers, or their classes, you will need to install fetchers' dependencies and their browser dependencies as follows: - ``` - pip install "scrapling[fetchers]" - scrapling install # normal install - scrapling install --force # force reinstall - ``` - This downloads all browsers, along with their system dependencies and fingerprint manipulation dependencies. - Or you can install them from the code instead of running a command like this: - ``` - from scrapling.cli import install - install([], standalone_mode=False) # normal install - install(["--force"], standalone_mode=False) # force reinstall - ``` -2. Extra features: - - Install the MCP server feature: - ``` - pip install "scrapling[ai]" - ``` - - Install dependencies for ([building RAG systems](https://scrapling.readthedocs.io/en/latest/ai/building-rag-systems.html)): - ``` - pip install "scrapling[rag]" - ``` - - Install shell features (Web Scraping shell and the `extract` command): - ``` - pip install "scrapling[shell]" - ``` - - Install everything: - ``` - pip install "scrapling[all]" - ``` - Remember that you need to install the browser dependencies with `scrapling install` after any of these extras (if you didn't already) - -### Docker - -You can also install a Docker image with all extras and browsers with the following command from DockerHub: - -``` -docker pull pyd4vinci/scrapling -``` - -Or download it from the GitHub registry: - -``` -docker pull ghcr.io/d4vinci/scrapling:latest -``` - -This image is automatically built and pushed using GitHub Actions and the repository's main branch. - -## Contributing - -We welcome contributions! Please read our [contributing guidelines](https://github.com/D4Vinci/Scrapling/blob/main/CONTRIBUTING.md) before getting started. - -## Disclaimer - -> [!caution] Caution -> This library is provided for educational and research purposes only. By using this library, you agree to comply with local and international data scraping and privacy laws. The authors and contributors are not responsible for any misuse of this software. Always respect the terms of service of websites and robots.txt files. - -## πŸŽ“ Citations - -If you have used our library for research purposes please quote us with the following reference: - -``` -@misc{scrapling, - author = {Karim Shoair}, - title = {Scrapling}, - year = {2024}, - url = {https://github.com/D4Vinci/Scrapling}, - note = {An adaptive Web Scraping framework that handles everything from a single request to a full-scale crawl!} -} -``` - -## License - -This work is licensed under the BSD-3-Clause License. - -## Acknowledgments - -This project includes code adapted from: - -- Parsel (BSD License)-Used for [translator](https://github.com/D4Vinci/Scrapling/blob/main/scrapling/core/translator.py) submodule - ---- - -Designed & crafted with ❀️ by Karim Shoair. \ No newline at end of file diff --git a/300 areas/395 AI Tools to try/Moonshot AI Launches Kimi Work, a Local Desktop Agent Reportedly Running on Kimi K2.6 With a 300-Sub-Agent Agent Swarm.md b/300 areas/395 AI Tools to try/Moonshot AI Launches Kimi Work, a Local Desktop Agent Reportedly Running on Kimi K2.6 With a 300-Sub-Agent Agent Swarm.md deleted file mode 100644 index f6ef1b8..0000000 --- a/300 areas/395 AI Tools to try/Moonshot AI Launches Kimi Work, a Local Desktop Agent Reportedly Running on Kimi K2.6 With a 300-Sub-Agent Agent Swarm.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -title: Moonshot AI Launches Kimi Work, a Local Desktop Agent Reportedly Running on Kimi K2.6 With a 300-Sub-Agent Agent Swarm -source: https://www.marktechpost.com/2026/06/12/moonshot-ai-launches-kimi-work-a-local-desktop-agent-reportedly-running-on-kimi-k2-6-with-a-300-sub-agent-agent-swarm/ -author: - - "[[Asif Razzaq]]" -published: 2026-06-12 -created: 2026-06-13 -description: "Moonshot AI launches Kimi Work: a local desktop agent pairing a 300-sub-agent Agent Swarm with WebBridge browser automation." -tags: - - clippings - - ai - - tool - - tools - - agents ---- -- [Editors Pick](https://www.marktechpost.com/category/editors-pick/) -- [Agentic AI](https://www.marktechpost.com/category/editors-pick/agentic-ai/) -- [AI Agents](https://www.marktechpost.com/category/editors-pick/ai-agents/) -- [Technology](https://www.marktechpost.com/category/technology/) -- [AI Shorts](https://www.marktechpost.com/category/technology/ai-shorts/) -- [Artificial Intelligence](https://www.marktechpost.com/category/technology/artificial-intelligence/) -- [Applications](https://www.marktechpost.com/category/technology/artificial-intelligence/applications/) -- [New Releases](https://www.marktechpost.com/category/editors-pick/new-releases/) -- [Software Engineering](https://www.marktechpost.com/category/editors-pick/software-engineering/) -- [Staff](https://www.marktechpost.com/category/editors-pick/staff/) -- [Tech News](https://www.marktechpost.com/category/tech-news/) - -Moonshot AI has introduced **[Kimi Work,](https://www.kimi.com/products/kimi-work)** an AI agent that runs on your own desktop. The Beijing-based AI entity announced it this week along with downloads for macOS and Windows. Kimi Work reads local files, drives your real browser, and runs scheduled tasks. It targets knowledge workers whose bottleneck is access to files and live sessions. - -Most agent tools of the past two years ran in the cloud. You type a goal, a remote server spins up a sandbox, and a hosted browser acts. Kimi Work runs locally instead, reaching files and sessions you already use. - -## What is Kimi Work? - -Kimi Work is a downloadable application, not a web chat. You give it goals in plain language, and it acts on your machine. Independent community mentions report that it runs on Kimi K2.6, Moonshot’s flagship model. - - - -K2.6 is an open-weight Mixture-of-Experts model released on April 20, 2026. It activates about 32 billion parameters per token. It carries a 256K-token context window for long, multi-step work. - -## How Kimi Work Operates - -Four building blocks define the product. Knowing them helps you reason about what it can do. - -1. **Agent Swarm:** Kimi Work can run many sub-agents in parallel on your machine. According to Moonshot release, the swarm scales to 300 sub-agents. The system splits a task into parts, then coordinates the results. K2.6’s swarm is documented up to 4,000 coordinated steps. -2. **WebBridge:** This browser extension lets the agent use a browser like a person. It searches, scrolls, extracts data, and fills forms across tabs. Because it uses your real session, it inherits your existing logins and cookies. -3. **Cron scheduling engine:** A built-in scheduler runs jobs on a daily, hourly, or conditional basis. Per Moonshot, triggers include LLM agent calls and Python or shell scripts. A β€œKeep Computer Awake” toggle keeps overnight jobs from stalling. -4. **Local files and code:** The agent reads folders you mount and runs Python in the background. According to Moonshot release, original files stay in place unless you approve a change. - -The desktop app also ships finance-specific data. It is pre-integrated with market data for A-shares, Hong Kong stocks, and US equities. According to Moonshot release, this removes the need for custom API setup. Finished research can convert into PowerPoint decks or Excel sheets. - -## Use Cases With Examples - -- **Document triage**: Point the agent at a folder of quarterly PDFs. Ask it to summarize them into one document, keeping originals intact. The swarm assigns one reader per file, then merges findings. -- **Web data collection**: Tell WebBridge to pull historical prices for three tickers. It opens your browser, sets the date range, and extracts the tables. Python then normalizes columns and writes an Excel workbook. -- **Scheduled briefings**: Define a 7:00 AM job in the cron engine. Each morning it gathers headlines and drafts a markdown briefing. With β€œKeep Computer Awake” on, the job survives overnight. -- **Office generation**: Ask for a short market-brief deck after a research pass. The agent drafts sections in parallel and renders native slides. - -## Kimi Work vs Cloud Agents - -The core difference is where the agent runs and what it can reach. The table compares Kimi Work against a typical cloud agent. - -| Dimension | Kimi Work (local) | Typical cloud agent | -| --- | --- | --- | -| Execution location | Your desktop | Vendor servers | -| File access | Mounts your local folders | Uploaded or sandboxed files | -| Browser | Your real, logged-in browser via WebBridge | Hosted virtual browser | -| Scheduling | Built-in cron engine | Often external or limited | -| Underlying model | Kimi K2.6, reported | Vendor’s hosted model | -| Setup | Install app, grant folder access | Zero-install, open a tab | -| Security responsibility | Falls on the user | Falls on the vendor | - -Neither approach wins outright. Local execution keeps data on your device and reaches real files. Cloud execution trades that control for zero-setup convenience and managed safety. - -## Scheduling: The Cron Engine in Practice - -Kimi Work is driven by natural language, not a public API. Its scheduler is a cron engine, so it accepts standard cron schedules. The five fields are: **minute, hour, day-of-month, month, and day-of-week.** - -```php -# Standard cron schedules the engine understands -0 7 * * * # every day at 07:00 -0 * * * * # every hour, on the hour -30 8 * * 1-5 # 08:30 on weekdays only (Mon-Fri) -0 0 1 * * # 00:00 on the first day of each month -``` - -You pair a schedule with a plain-language task. A daily briefing job reads like this. - -```php -Schedule: 0 7 * * * (every day at 07:00) -Task: "Draft today's market briefing and save it to - ~/KimiWorkspace/briefing.md. Ask before writing." -``` - -The approval gate then applies to that write, and to any web action. - -## Key Takeaways - -- An β€œAsk before acting” gate, with YOLO mode off, prompts before any file write. -- Kimi Work is a local desktop agent for macOS (Apple silicon) and Windows. -- An Agent Swarm runs up to 300 sub-agents in parallel on your machine. -- WebBridge drives your logged-in browser; a built-in cron engine runs scheduled jobs. -- It reads local folders and runs Python, keeping originals unless you approve changes. - ---- - -Check out the **[Details here](https://www.kimi.com/products/kimi-work).** Also, feel free to follow us on **[==Twitter==](https://x.com/intent/follow?screen_name=marktechpost)** and don’t forget to join our **[150k+ML SubReddit](https://www.reddit.com/r/machinelearningnews/)** and Subscribe to **[our Newsletter](https://www.aidevsignals.com/)**. Wait! are you on telegram? **[now you can join us on telegram as well.](https://t.me/machinelearningresearchnews)** - -Need to partner with us for promoting your GitHub Repo OR Hugging Face Page OR Product Release OR Webinar etc.? **[==Connect with us==](https://forms.gle/wbash1wF6efRj8G58)** \ No newline at end of file