SearXNG instances frequently get blocked by search engines, causing Hermes agents to lose search capability entirely. This tutorial adds an API fallback that activates when SearXNG is blocked, rate-limited, or returning errors. The agent keeps working regardless of SearXNG status, and the API fallback costs $0.005/query.
Prerequisites
- Python 3.8+
- requests library
- A Scavio API key from scavio.dev
- Hermes agent with SearXNG configured
Walkthrough
Step 1: Detect SearXNG failures
Build a health checker that detects when SearXNG is blocked or failing.
Python
import os, requests, time, json
from datetime import datetime
SEARXNG_URL = os.environ.get('SEARXNG_URL', 'http://localhost:8888')
API_KEY = os.environ['SCAVIO_API_KEY']
SH = {'x-api-key': API_KEY, 'Content-Type': 'application/json'}
def check_searxng_health():
"""Check if SearXNG is responsive and returning results."""
try:
resp = requests.get(f'{SEARXNG_URL}/search',
params={'q': 'test', 'format': 'json'}, timeout=5)
if resp.status_code != 200:
return {'healthy': False, 'reason': f'HTTP {resp.status_code}'}
data = resp.json()
if not data.get('results'):
return {'healthy': False, 'reason': 'No results (likely blocked)'}
return {'healthy': True, 'results': len(data['results'])}
except requests.exceptions.ConnectionError:
return {'healthy': False, 'reason': 'Connection refused (SearXNG down)'}
except requests.exceptions.Timeout:
return {'healthy': False, 'reason': 'Timeout (SearXNG overloaded)'}
except Exception as e:
return {'healthy': False, 'reason': str(e)[:50]}
def search_api_fallback(query, num_results=5):
"""Scavio API fallback when SearXNG fails."""
resp = requests.post('https://api.scavio.dev/api/v1/search',
headers=SH, json={'query': query, 'country_code': 'us'}, timeout=10)
resp.raise_for_status()
data = resp.json()
return [{'title': r.get('title', ''), 'url': r.get('link', ''),
'content': r.get('snippet', '')} for r in data.get('organic_results', [])[:num_results]]
health = check_searxng_health()
print(f'SearXNG status: {"healthy" if health["healthy"] else "UNHEALTHY"}')
if not health['healthy']:
print(f' Reason: {health["reason"]}')
print(f' Fallback: Scavio API ready ($0.005/query)')Step 2: Build the smart search router
Route searches to SearXNG when healthy, API when not.
Python
class SmartSearchRouter:
def __init__(self):
self.searxng_failures = 0
self.last_check = 0
self.check_interval = 300 # Re-check SearXNG every 5 min
self.api_calls = 0
def search(self, query, num_results=5):
# Try SearXNG first if it has been healthy recently
if self.searxng_failures < 3 or time.time() - self.last_check > self.check_interval:
try:
resp = requests.get(f'{SEARXNG_URL}/search',
params={'q': query, 'format': 'json', 'engines': 'google'},
timeout=5)
if resp.status_code == 200:
data = resp.json()
results = data.get('results', [])
if results:
self.searxng_failures = 0
return {'provider': 'searxng', 'results': [
{'title': r.get('title', ''), 'url': r.get('url', ''),
'content': r.get('content', '')}
for r in results[:num_results]
]}
self.searxng_failures += 1
self.last_check = time.time()
except Exception:
self.searxng_failures += 1
self.last_check = time.time()
# Fallback to API
results = search_api_fallback(query, num_results)
self.api_calls += 1
return {'provider': 'scavio_api', 'results': results}
router = SmartSearchRouter()
# Simulate searches
for q in ['python async tutorial', 'FastAPI deployment', 'database migration tools']:
result = router.search(q)
print(f' [{result["provider"]:12}] {q:30} | {len(result["results"])} results')
print(f'\nAPI fallback calls: {router.api_calls}')
print(f'API cost so far: ${router.api_calls * 0.005:.3f}')Step 3: Replace Hermes search with the smart router
Swap the Hermes search tool to use the smart router instead of direct SearXNG.
Python
def patch_hermes_search():
"""Replace Hermes SearXNG search with smart router."""
router = SmartSearchRouter()
def hermes_search_tool(query, num_results=5):
"""Drop-in replacement for Hermes search tool."""
result = router.search(query, num_results)
# Format in SearXNG response style for compatibility
return {
'results': result['results'],
'number_of_results': len(result['results']),
'_provider': result['provider'],
}
return hermes_search_tool
# Apply the patch
search = patch_hermes_search()
# Test with simulated SearXNG failure
print('Testing with SearXNG down...')
result = search('how to fix SearXNG blocking')
print(f' Provider: {result["_provider"]}')
print(f' Results: {result["number_of_results"]}')
for r in result['results'][:3]:
print(f' {r["title"][:50]}')
print(f'\n SearXNG blocking fix:')
print(f' Problem: SearXNG gets blocked by Google, returns 0 results')
print(f' Solution: Smart router falls back to Scavio API')
print(f' Cost: $0.005/query (only when SearXNG is down)')
print(f' Latency: ~300ms API vs 2-5s SearXNG')Python Example
Python
import os, requests
SH = {'x-api-key': os.environ['SCAVIO_API_KEY'], 'Content-Type': 'application/json'}
def search_fallback(query):
"""Use when SearXNG is blocked."""
data = requests.post('https://api.scavio.dev/api/v1/search',
headers=SH, json={'query': query, 'country_code': 'us'}, timeout=10).json()
return [{'title': r['title'], 'url': r['link'], 'content': r.get('snippet', '')}
for r in data.get('organic_results', [])[:5]]
for r in search_fallback('SearXNG alternative'):
print(f'{r["title"][:50]}')
print('SearXNG fallback active. $0.005/query.')JavaScript Example
JavaScript
const SH = { 'x-api-key': process.env.SCAVIO_API_KEY, 'Content-Type': 'application/json' };
// Fallback when SearXNG is blocked
const data = await fetch('https://api.scavio.dev/api/v1/search', {
method: 'POST', headers: SH,
body: JSON.stringify({ query: 'SearXNG alternative', country_code: 'us' })
}).then(r => r.json());
(data.organic_results || []).slice(0, 5).forEach(r => console.log(r.title));
console.log('SearXNG fallback ready.');Expected Output
JSON
SearXNG status: UNHEALTHY
Reason: Connection refused (SearXNG down)
Fallback: Scavio API ready ($0.005/query)
[scavio_api ] python async tutorial | 5 results
[scavio_api ] FastAPI deployment | 5 results
[scavio_api ] database migration tools | 5 results
API fallback calls: 3
API cost so far: $0.015
Testing with SearXNG down...
Provider: scavio_api
Results: 5
SearXNG blocking fix:
Problem: SearXNG gets blocked by Google, returns 0 results
Solution: Smart router falls back to Scavio API
Cost: $0.005/query (only when SearXNG is down)
Latency: ~300ms API vs 2-5s SearXNG