SearXNG is an open-source meta-search engine that aggregates results from Google, Bing, DuckDuckGo, and others. It works well for personal use but breaks under production load because upstream engines block IPs aggressively. This tutorial deploys SearXNG behind rotating proxies with health checks and automatic fallback to Scavio when SearXNG returns empty results. You get free search for most queries and pay $0.005 only when fallback triggers.
Prerequisites
- Docker and Docker Compose installed
- A rotating proxy service (e.g., BrightData, Oxylabs)
- Python 3.9+ installed
- A Scavio API key for fallback
Walkthrough
Step 1: Deploy SearXNG with Docker Compose
Set up SearXNG with Redis for rate limiting and a custom settings file that configures engines and output format.
Python
# docker-compose.yml
# services:
# searxng:
# image: searxng/searxng:latest
# ports:
# - '8888:8080'
# volumes:
# - ./searxng:/etc/searxng
# environment:
# - SEARXNG_BASE_URL=http://localhost:8888
# redis:
# image: redis:alpine
# ports:
# - '6379:6379'
# searxng/settings.yml (key sections):
# server:
# limiter: true
# secret_key: "change-me-to-random-string"
# outgoing:
# proxies:
# all://:
# - socks5h://user:pass@proxy1:1080
# - socks5h://user:pass@proxy2:1080
# engines:
# - name: google
# shortcut: g
# disabled: false
# - name: bing
# shortcut: b
# disabled: false
import subprocess
result = subprocess.run(['docker', 'compose', 'up', '-d'], capture_output=True, text=True)
print(result.stdout or 'SearXNG started')
print('Access: http://localhost:8888')Step 2: Build the SearXNG client with health checks
Create a client that queries SearXNG JSON API, detects failures (empty results, timeouts, blocked responses), and tracks uptime.
Python
import requests, time
from collections import deque
class SearXNGClient:
def __init__(self, base_url='http://localhost:8888'):
self.base_url = base_url
self.health_window = deque(maxlen=20) # last 20 queries
def search(self, query: str, count: int = 10) -> list:
try:
resp = requests.get(f'{self.base_url}/search', params={
'q': query, 'format': 'json', 'categories': 'general'
}, timeout=15)
if resp.status_code != 200:
self.health_window.append(False)
return []
results = resp.json().get('results', [])
self.health_window.append(len(results) > 0)
return [{'title': r.get('title', ''), 'link': r.get('url', ''),
'snippet': r.get('content', '')} for r in results[:count]]
except requests.exceptions.RequestException:
self.health_window.append(False)
return []
@property
def health_pct(self) -> float:
if not self.health_window:
return 100.0
return sum(self.health_window) / len(self.health_window) * 100
searxng = SearXNGClient()
results = searxng.search('python tutorial')
print(f'Results: {len(results)}, Health: {searxng.health_pct:.0f}%')Step 3: Add Scavio fallback with automatic failover
When SearXNG health drops below 50% or returns no results, automatically route to Scavio. Log fallback events for monitoring.
Python
import os
SCAVIO_KEY = os.environ['SCAVIO_API_KEY']
def search_with_fallback(query: str, count: int = 10) -> dict:
# Skip SearXNG if health is poor
if searxng.health_pct < 50:
print(f'SearXNG health {searxng.health_pct:.0f}% -- routing to Scavio')
return {'results': scavio_search(query, count), 'provider': 'scavio', 'cost': 0.005}
results = searxng.search(query, count)
if results:
return {'results': results, 'provider': 'searxng', 'cost': 0}
# Fallback
print(f'SearXNG returned 0 results -- falling back to Scavio')
return {'results': scavio_search(query, count), 'provider': 'scavio', 'cost': 0.005}
def scavio_search(query: str, count: int = 10) -> list:
resp = requests.post('https://api.scavio.dev/api/v1/search',
headers={'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json'},
json={'query': query, 'country_code': 'us', 'num_results': count})
resp.raise_for_status()
return [{'title': r['title'], 'link': r['link'],
'snippet': r.get('snippet', '')} for r in resp.json().get('organic_results', [])]
result = search_with_fallback('best search api 2026')
print(f'Provider: {result["provider"]}, Results: {len(result["results"])}, Cost: ${result["cost"]}')Python Example
Python
import requests, os
from collections import deque
SCAVIO_KEY = os.environ['SCAVIO_API_KEY']
health = deque(maxlen=20)
def searxng(query, count=10):
try:
resp = requests.get('http://localhost:8888/search',
params={'q': query, 'format': 'json'}, timeout=15)
results = resp.json().get('results', [])[:count]
health.append(len(results) > 0)
return [{'title': r['title'], 'link': r['url'], 'snippet': r.get('content', '')} for r in results]
except Exception:
health.append(False)
return []
def search(query, count=10):
health_pct = sum(health) / len(health) * 100 if health else 100
if health_pct >= 50:
results = searxng(query, count)
if results:
return results
resp = requests.post('https://api.scavio.dev/api/v1/search',
headers={'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json'},
json={'query': query, 'country_code': 'us', 'num_results': count})
return [{'title': r['title'], 'link': r['link'], 'snippet': r.get('snippet', '')}
for r in resp.json().get('organic_results', [])]
for r in search('best python frameworks 2026'):
print(r['title'])JavaScript Example
JavaScript
const SCAVIO_KEY = process.env.SCAVIO_API_KEY;
const health = [];
async function searxng(query, count = 10) {
try {
const resp = await fetch(`http://localhost:8888/search?q=${encodeURIComponent(query)}&format=json`);
const results = (await resp.json()).results?.slice(0, count) || [];
health.push(results.length > 0);
if (health.length > 20) health.shift();
return results.map(r => ({ title: r.title, link: r.url, snippet: r.content }));
} catch { health.push(false); return []; }
}
async function search(query, count = 10) {
const pct = health.length ? health.filter(Boolean).length / health.length * 100 : 100;
if (pct >= 50) { const r = await searxng(query, count); if (r.length) return r; }
const resp = await fetch('https://api.scavio.dev/api/v1/search', {
method: 'POST',
headers: { 'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json' },
body: JSON.stringify({ query, country_code: 'us', num_results: count })
});
return (await resp.json()).organic_results?.map(r => ({ title: r.title, link: r.link, snippet: r.snippet })) || [];
}
search('best python frameworks 2026').then(r => r.forEach(x => console.log(x.title)));Expected Output
JSON
SearXNG returned 0 results -- falling back to Scavio
Provider: scavio, Results: 10, Cost: $0.005
SearXNG health: 45% (9/20 successful)
Fallback rate: 55% of queries routed to Scavio
Estimated monthly cost at 1000 queries: $2.75