Common Crawl proporciona petabytes de datos de archivos web gratuitos, pero tiene meses y requiere un procesamiento intenso. Las API de búsqueda en vivo arrojan resultados nuevos al instante pero con un costo por consulta. Este tutorial combina ambos en una canalización híbrida: use Common Crawl para un análisis histórico masivo y Scavio para una actualización en tiempo real. El enfoque híbrido le brinda la escala de Common Crawl con la moneda de la búsqueda en vivo a un costo mínimo.
Requisitos previos
- Python 3.9+ instalado
- solicita biblioteca instalada
- Una clave API de Scavio de scavio.dev
- Comprensión básica del índice de rastreo común
Guia paso a paso
Paso 1: Consultar la API del índice de rastreo común
Busque en el índice CDX de Common Crawl para encontrar páginas web históricas que coincidan con su consulta. El índice se puede consultar de forma gratuita y devuelve URL de páginas archivadas.
Python
import requests
CC_INDEX_URL = 'https://index.commoncrawl.org/CC-MAIN-2026-04-index'
def search_common_crawl(domain: str, limit: int = 20) -> list:
"""Search Common Crawl index for pages from a domain."""
resp = requests.get(CC_INDEX_URL,
params={'url': f'{domain}/*', 'output': 'json', 'limit': limit},
timeout=30)
if resp.status_code != 200:
print(f'Common Crawl returned {resp.status_code}')
return []
lines = resp.text.strip().split('\n')
results = []
for line in lines:
try:
import json
record = json.loads(line)
results.append({
'url': record.get('url', ''),
'timestamp': record.get('timestamp', ''),
'status': record.get('status', ''),
'mime': record.get('mime', ''),
'length': record.get('length', 0),
})
except json.JSONDecodeError:
continue
return results
# Search for pages from a domain
pages = search_common_crawl('docs.python.org')
print(f'Found {len(pages)} archived pages from docs.python.org')
for p in pages[:5]:
print(f' [{p["timestamp"]}] {p["url"][:60]}')Paso 2: Construya la función de búsqueda híbrida
Combine datos de rastreo común con resultados de búsqueda en vivo. Utilice Common Crawl para cobertura histórica y búsqueda en vivo para obtener resultados nuevos.
Python
import os
SCAVIO_KEY = os.environ['SCAVIO_API_KEY']
def live_search(query: str, count: int = 5) -> list:
resp = requests.post('https://api.scavio.dev/api/v1/search',
headers={'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json'},
json={'query': query, 'country_code': 'us', 'num_results': count})
return [{'url': r['link'], 'title': r['title'],
'snippet': r.get('snippet', ''), 'source': 'live_search'}
for r in resp.json().get('organic_results', [])]
def hybrid_search(query: str, domain: str = None) -> dict:
"""Combine Common Crawl historical data with live results."""
results = {'historical': [], 'live': [], 'cost': 0}
# Step 1: Check Common Crawl for historical data (free)
if domain:
cc_results = search_common_crawl(domain, limit=10)
results['historical'] = [{
'url': r['url'], 'timestamp': r['timestamp'],
'source': 'common_crawl'
} for r in cc_results]
# Step 2: Get fresh live results ($0.005)
live = live_search(query, count=5)
results['live'] = live
results['cost'] = 0.005
results['total'] = len(results['historical']) + len(results['live'])
return results
result = hybrid_search('python 3.14 new features', domain='docs.python.org')
print(f'Historical (Common Crawl): {len(result["historical"])} pages')
print(f'Live (Scavio): {len(result["live"])} results')
print(f'Total: {result["total"]} | Cost: ${result["cost"]}')Paso 3: Analizar las brechas de cobertura entre datos históricos y en vivo
Compare lo que tiene Common Crawl con lo que devuelve la búsqueda en vivo. Esto revela contenido que existe pero que no está en el archivo y contenido archivado que puede haber sido eliminado.
Python
def analyze_coverage(domain: str, queries: list) -> dict:
# Get Common Crawl inventory
cc_urls = set(r['url'] for r in search_common_crawl(domain, limit=50))
# Get live search URLs for the domain
live_urls = set()
for query in queries:
live = live_search(f'site:{domain} {query}', count=5)
live_urls.update(r['url'] for r in live)
# Analyze overlap
both = cc_urls & live_urls
cc_only = cc_urls - live_urls
live_only = live_urls - cc_urls
print(f'Domain: {domain}')
print(f'Common Crawl pages: {len(cc_urls)}')
print(f'Live search pages: {len(live_urls)}')
print(f'In both: {len(both)}')
print(f'CC only (may be removed): {len(cc_only)}')
print(f'Live only (new content): {len(live_only)}')
print(f'Cost: {len(queries)} live searches = ${len(queries) * 0.005:.3f}')
return {'cc_only': cc_only, 'live_only': live_only, 'both': both}
coverage = analyze_coverage('docs.python.org', ['asyncio', 'pathlib', 'dataclasses'])Ejemplo en Python
Python
import requests, os, json
SCAVIO_KEY = os.environ['SCAVIO_API_KEY']
CC_INDEX = 'https://index.commoncrawl.org/CC-MAIN-2026-04-index'
def hybrid_search(query, domain=None):
results = []
# Common Crawl (free)
if domain:
cc = requests.get(CC_INDEX, params={'url': f'{domain}/*', 'output': 'json', 'limit': 10}, timeout=30)
for line in cc.text.strip().split('\n'):
try:
r = json.loads(line)
results.append({'url': r['url'], 'source': 'common_crawl'})
except Exception:
pass
# Live search ($0.005)
live = requests.post('https://api.scavio.dev/api/v1/search',
headers={'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json'},
json={'query': query, 'country_code': 'us', 'num_results': 5})
for r in live.json().get('organic_results', []):
results.append({'url': r['link'], 'title': r['title'], 'source': 'live'})
print(f'{len(results)} total results (cost: $0.005)')
return results
hybrid_search('python asyncio', 'docs.python.org')Ejemplo en JavaScript
JavaScript
const SCAVIO_KEY = process.env.SCAVIO_API_KEY;
async function hybridSearch(query, domain) {
const results = [];
if (domain) {
const cc = await fetch(`https://index.commoncrawl.org/CC-MAIN-2026-04-index?url=${domain}/*&output=json&limit=10`);
const lines = (await cc.text()).trim().split('\n');
lines.forEach(line => { try { results.push({...JSON.parse(line), source: 'cc'}); } catch {} });
}
const live = await fetch('https://api.scavio.dev/api/v1/search', {
method: 'POST',
headers: { 'x-api-key': SCAVIO_KEY, 'Content-Type': 'application/json' },
body: JSON.stringify({ query, country_code: 'us', num_results: 5 })
});
for (const r of (await live.json()).organic_results || []) {
results.push({ url: r.link, title: r.title, source: 'live' });
}
console.log(`${results.length} results (cost: $0.005)`);
return results;
}
hybridSearch('python asyncio', 'docs.python.org');Salida esperada
JSON
Found 15 archived pages from docs.python.org
[20260115] https://docs.python.org/3/library/asyncio.html
[20260115] https://docs.python.org/3/whatsnew/3.14.html
Historical (Common Crawl): 15 pages
Live (Scavio): 5 results
Total: 20 | Cost: $0.005
Domain: docs.python.org
Common Crawl pages: 50
Live search pages: 12
In both: 8
CC only (may be removed): 42
Live only (new content): 4
Cost: 3 live searches = $0.015