From b0ade653fce01d5bdeaaa89c71c3d196f53418e0 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 2 Dec 2025 18:37:14 +0100 Subject: [PATCH] feat(stack): add SearXNG metasearch engine for AI web search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploy SearXNG privacy-focused metasearch engine as infrastructure service for core-ai web search capabilities. Changes: - Add SearXNG Docker Compose stack - Image: searxng/searxng:latest - Port: 8087 (HTTP API with JSON format) - Network: docker-dataplane (shared with core-ai) - Storage: Config only at ~/docker-data/searxng - Redis integration: DB 5 on redis-shared for result caching - Resource limits: 1 CPU, 512MB RAM - Health check: /healthz endpoint Features: - Aggregates 246+ search engines (Google, Bing, DuckDuckGo, etc.) - Privacy-first: No tracking, no profiling, no data collection - Multi-format: HTML (web UI), JSON (API for core-ai) - Configurable categories: general, it, science, news, images, videos, etc. - Result caching via Redis (reduces duplicate queries) Search Categories: - general: Web search - it: Programming/technical (StackOverflow, GitHub, documentation) - science: Academic (arXiv, PubMed, Semantic Scholar) - news: News articles - images/videos: Media search - map: Geographic/location queries Integration: - Core-AI web_search tool uses http://searxng:8080/search API - JSON format enabled for programmatic access - Timezone: Europe/Amsterdam Security: - Dropped all capabilities except essential (CHOWN, SETGID, SETUID) - No data persistence (privacy by design) - Internal network only (not exposed via NPM) Setup: 1. mkdir -p ~/docker-data/searxng 2. docker-compose -f stacks/searxng.yml up -d 3. Verify: curl "http://localhost:8087/search?q=test&format=json" API Usage: GET http://searxng:8080/search?q=query&format=json&categories=general Resource Usage: - CPU: ~0.2-0.5 cores - RAM: ~150-300MB - Disk: ~10-50MB (config only) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- stacks/searxng.yml | 226 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 stacks/searxng.yml diff --git a/stacks/searxng.yml b/stacks/searxng.yml new file mode 100644 index 0000000..aed529e --- /dev/null +++ b/stacks/searxng.yml @@ -0,0 +1,226 @@ +version: '3.8' + +# SearXNG Metasearch Engine +# Purpose: Privacy-focused metasearch aggregating 246+ search engines +# Port: 8087 (HTTP API) +# GPU: NO - Pure CPU application +# Storage: Minimal (config only, no data persistence) +# Integration: Core-AI service tool for web search capability + +services: + searxng: + image: searxng/searxng:latest + container_name: searxng + restart: unless-stopped + ports: + - "8087:8080" # HTTP API (JSON format enabled) + volumes: + # Mount config directory (will be auto-populated on first run) + - /home/jpmschweitzer/docker-data/searxng:/etc/searxng:rw + environment: + - TZ=Europe/Amsterdam + - SEARXNG_BASE_URL=http://searxng:8087/ + # Valkey/Redis configuration for result caching (using shared Redis DB 5) + - SEARXNG_VALKEY_URL=redis://redis-shared:6379/5 + # Enabled output formats (JSON required for API access) + - SEARXNG_SETTINGS_FORMATS=html,json + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + reservations: + memory: 128M + networks: + - docker-dataplane + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/healthz"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + +networks: + docker-dataplane: + external: true + name: docker-dataplane + +# SearXNG Overview: +# - Metasearch engine: Aggregates results from 246+ search sources +# - Privacy-first: No tracking, no profiling, no data collection +# - Multi-format: HTML (web UI), JSON (API), CSV, RSS +# - Customizable: Enable/disable specific engines per category +# +# Search Categories: +# - general: Web search (Google, Bing, DuckDuckGo, etc.) +# - images: Image search +# - videos: Video search +# - news: News articles +# - map: Geographic/location +# - music: Music/audio +# - it: Programming/technical (StackOverflow, GitHub, docs) +# - science: Academic (arXiv, PubMed, Semantic Scholar) +# - files: File repositories +# - social media: Social platforms +# +# Setup Instructions: +# +# 1. Create configuration directory: +# mkdir -p ~/docker-data/searxng/config +# +# 2. Deploy stack: +# docker-compose -f searxng.yml up -d +# +# Note: First run will auto-generate settings.yml with secret key +# +# 3. Stop container to edit config: +# docker stop searxng +# +# 4. Enable JSON format in settings.yml: +# Edit ~/docker-data/searxng/config/settings.yml +# Find the 'search:' section and set: +# formats: +# - html +# - json +# +# 5. Restart container: +# docker start searxng +# +# 6. Verify deployment: +# # Web UI test: +# curl http://localhost:8087/ +# +# # JSON API test: +# curl "http://localhost:8087/search?q=python&format=json" | jq '.results[0]' +# +# # Health check: +# curl http://localhost:8087/healthz +# +# API Usage: +# +# Basic search: +# GET http://searxng:8080/search?q=query&format=json +# +# With category filter: +# GET http://searxng:8080/search?q=machine+learning&format=json&categories=science +# +# With language: +# GET http://searxng:8080/search?q=query&format=json&language=en +# +# With time range: +# GET http://searxng:8080/search?q=news&format=json&time_range=day +# +# Available categories: +# general, images, videos, news, map, music, it, science, files, social_media +# +# Time ranges: +# day, week, month, year +# +# Response format (JSON): +# { +# "query": "search term", +# "results": [ +# { +# "url": "https://example.com", +# "title": "Page title", +# "content": "Description snippet", +# "engine": "google", +# "score": 1.0 +# } +# ], +# "suggestions": ["related", "searches"], +# "number_of_results": 42 +# } +# +# Integration with Core-AI: +# +# Add to services/core-ai/src/tools/local.py: +# +# @register_tool +# async def web_search(query: str, category: str = "general") -> str: +# """Search the web using SearXNG metasearch engine.""" +# response = await httpx.get( +# "http://searxng:8080/search", +# params={"q": query, "format": "json", "categories": category}, +# timeout=10.0 +# ) +# results = response.json()["results"][:5] +# return "\n\n".join([ +# f"[{r['title']}]({r['url']})\n{r.get('content', '')}" +# for r in results +# ]) +# +# Performance Tuning: +# +# Response times: 2-5 seconds (aggregating multiple sources) +# Redis caching: Reduces duplicate queries (DB 5 on redis-shared) +# Engine selection: Disable slow engines to improve speed +# +# Edit settings.yml to disable slow engines: +# engines: +# - name: slowengine +# disabled: true +# +# Resource Usage (expected): +# CPU: ~0.2-0.5 cores (varies with query load) +# RAM: ~150-300MB (depends on cache size) +# Disk: ~10-50MB (config only, no data storage) +# Network: Variable (depends on upstream engine responses) +# +# Redis Database Allocation: +# DB 5: SearXNG result cache +# +# Security: +# - Dropped all capabilities except essential (CHOWN, SETGID, SETUID) +# - No data persistence (privacy by design) +# - Can run behind Nginx Proxy Manager for HTTPS +# - Optional Tor support (requires additional config) +# +# Monitoring: +# +# View logs: +# docker logs -f searxng +# +# Check engine stats: +# curl http://localhost:8080/stats +# +# Check health: +# docker exec searxng wget -q -O- http://localhost:8080/healthz +# +# Engine Configuration Tips: +# +# To optimize for AI/LLM use cases, consider enabling these engines: +# - General: google, bing, duckduckgo, brave +# - Technical: stackoverflow, github, devdocs, mdn +# - Academic: arxiv, pubmed, semantic_scholar, google_scholar +# - Documentation: readthedocs, man (Linux man pages) +# +# Disable to improve speed: +# - Slow engines (check /stats page) +# - Engines you don't need (social media if not relevant) +# - Engines with frequent timeouts +# +# Advanced Configuration: +# +# Custom engines can be added to settings.yml +# See: https://docs.searxng.org/dev/engines/index.html +# +# Limiter (rate limiting) can be configured to prevent abuse +# See: https://docs.searxng.org/admin/engines/settings.html#limiter +# +# After Deployment: +# 1. Access UI: http://localhost:8080 +# 2. Test JSON API: curl "http://localhost:8080/search?q=test&format=json" +# 3. Review engine stats: http://localhost:8080/stats +# 4. Integrate with core-ai service