Karpathy's LLM Wiki project sparked discussion on r/AI_Agents about ingestion pipelines. Most wiki builders use separate tools for Google, Reddit, YouTube, and Amazon data. This tutorial replaces them all with a single API that covers multiple platforms, reducing integration complexity from 5 SDKs to 1.
Prerequisites
- Scavio API key
- Python 3.8+
- Markdown-based wiki (git repo or filesystem)
Walkthrough
Step 1: Define wiki topic research function
One function searches all platforms for a topic.
Python
import requests, os
# Scavio has one endpoint per platform - there is no dispatcher endpoint and no
# `platform` request param, so the selector lives in your code.
SCAVIO = "https://api.scavio.dev"
SCAVIO_ENDPOINTS = {
"google": "/api/v2/google",
"reddit": "/api/v1/reddit/search",
"youtube": "/api/v1/youtube/search",
"amazon": "/api/v1/amazon/search",
"walmart": "/api/v1/walmart/search",
}
SCAVIO_QUERY_KEY = {"youtube": "search"}
SCAVIO_RESULTS_KEY = {"google": "organic_results", "reddit": "results",
"youtube": "results", "amazon": "products", "walmart": "products"}
def scavio_url(platform):
return SCAVIO + SCAVIO_ENDPOINTS[platform or "google"]
def scavio_body(platform, query):
return {SCAVIO_QUERY_KEY.get(platform or "google", "query"): query}
def scavio_payload(payload, platform="google"):
"""Google v2 passes Google's response through as-is; every other endpoint
wraps its payload in `data`. Item fields differ per platform (see
https://scavio.dev/docs), so only the result list is normalised here."""
platform = platform or "google"
out = payload if platform == "google" else payload["data"]
return {**out, "results": out.get(SCAVIO_RESULTS_KEY[platform], [])}
H = {'Authorization': 'Bearer ' + os.environ['SCAVIO_API_KEY']}
def research_topic(topic):
sources = {}
for platform in ['google', 'reddit', 'youtube']:
data = scavio_payload(requests.post(scavio_url(platform), headers=H, json=scavio_body(platform, topic)).json(), platform)
sources[platform] = data.get('results', []) or data.get('results', [])
return sourcesStep 2: Generate wiki entry from multi-platform data
Combine search results into a structured wiki page.
Python
def generate_wiki_entry(topic, sources):
entry = f'# {topic}\n\n'
entry += '## Overview\n'
# Use top Google results for the overview
for r in sources.get('google', [])[:3]:
entry += f"- [{r.get('title', '')}]({r.get('link', '')}): {r.get('snippet', '')}\n"
entry += '\n## Community Discussion\n'
for r in sources.get('reddit', [])[:3]:
entry += f"- [{r.get('title', '')}]({r.get('url', '')})\n"
entry += '\n## Video Resources\n'
for r in sources.get('youtube', [])[:3]:
entry += f"- {r.get('title', '')}\n"
return entryStep 3: Build the wiki in batch
Process a list of topics and save as markdown files.
Python
import os
def build_wiki(topics, output_dir='wiki'):
os.makedirs(output_dir, exist_ok=True)
for topic in topics:
slug = topic.lower().replace(' ', '-')
sources = research_topic(topic)
entry = generate_wiki_entry(topic, sources)
with open(f'{output_dir}/{slug}.md', 'w') as f:
f.write(entry)
print(f'Built wiki page: {slug}.md')
topics = ['transformer architecture', 'RLHF training', 'RAG pipeline', 'MCP protocol']
build_wiki(topics)Step 4: Add freshness checks
Re-research topics that are older than 7 days.
Python
import datetime
def needs_refresh(filepath, max_age_days=7):
if not os.path.exists(filepath):
return True
mtime = datetime.datetime.fromtimestamp(os.path.getmtime(filepath))
return (datetime.datetime.now() - mtime).days > max_age_days
def refresh_wiki(topics, output_dir='wiki'):
for topic in topics:
slug = topic.lower().replace(' ', '-')
path = f'{output_dir}/{slug}.md'
if needs_refresh(path):
sources = research_topic(topic)
entry = generate_wiki_entry(topic, sources)
with open(path, 'w') as f:
f.write(entry)
print(f'Refreshed: {slug}')Python Example
Python
import os, requests
# Scavio has one endpoint per platform - there is no dispatcher endpoint and no
# `platform` request param, so the selector lives in your code.
SCAVIO = "https://api.scavio.dev"
SCAVIO_ENDPOINTS = {
"google": "/api/v2/google",
"reddit": "/api/v1/reddit/search",
"youtube": "/api/v1/youtube/search",
"amazon": "/api/v1/amazon/search",
"walmart": "/api/v1/walmart/search",
}
SCAVIO_QUERY_KEY = {"youtube": "search"}
SCAVIO_RESULTS_KEY = {"google": "organic_results", "reddit": "results",
"youtube": "results", "amazon": "products", "walmart": "products"}
def scavio_url(platform):
return SCAVIO + SCAVIO_ENDPOINTS[platform or "google"]
def scavio_body(platform, query):
return {SCAVIO_QUERY_KEY.get(platform or "google", "query"): query}
def scavio_payload(payload, platform="google"):
"""Google v2 passes Google's response through as-is; every other endpoint
wraps its payload in `data`. Item fields differ per platform (see
https://scavio.dev/docs), so only the result list is normalised here."""
platform = platform or "google"
out = payload if platform == "google" else payload["data"]
return {**out, "results": out.get(SCAVIO_RESULTS_KEY[platform], [])}
H = {'Authorization': 'Bearer ' + os.environ['SCAVIO_API_KEY']}
def wiki_page(topic):
data = {}
for p in ['google', 'reddit', 'youtube']:
r = scavio_payload(requests.post(scavio_url(p), headers=H, json=scavio_body(p, topic)).json(), p)
data[p] = r.get('results', []) or r.get('results', [])
return data
# 4 topics x 3 platforms = 12 queries = $0.06JavaScript Example
JavaScript
// Scavio has one endpoint per platform - there is no dispatcher endpoint and no
// `platform` request param, so the selector lives in your code.
const SCAVIO = "https://api.scavio.dev";
const SCAVIO_ENDPOINTS = {
google: "/api/v2/google",
reddit: "/api/v1/reddit/search",
youtube: "/api/v1/youtube/search",
amazon: "/api/v1/amazon/search",
walmart: "/api/v1/walmart/search",
};
const SCAVIO_QUERY_KEY = { youtube: "search" };
const SCAVIO_RESULTS_KEY = { google: "organic_results", reddit: "results",
youtube: "results", amazon: "products", walmart: "products" };
const scavioUrl = (platform) => SCAVIO + SCAVIO_ENDPOINTS[platform || "google"];
const scavioBody = (platform, query) =>
({ [SCAVIO_QUERY_KEY[platform || "google"] || "query"]: query });
// Google v2 passes Google's response through as-is; every other endpoint wraps
// its payload in `data`. Item fields differ per platform (see
// https://scavio.dev/docs), so only the result list is normalised here.
function scavioPayload(json, platform = "google") {
const p = platform || "google";
const out = p === "google" ? json : json.data;
return { ...out, results: out[SCAVIO_RESULTS_KEY[p]] || [] };
}
const platforms = ['google', 'reddit', 'youtube'];
const sources = {};
for (const platform of platforms) {
const res = await fetch(scavioUrl("google"), {
method: 'POST',
headers: {'Authorization': `Bearer ${process.env.SCAVIO_API_KEY}`, 'Content-Type': 'application/json'},
body: JSON.stringify(scavioBody("google", topic))
});
sources[platform] = scavioPayload(await res.json(), "google");
}Expected Output
JSON
Markdown wiki with multi-platform sources per topic. One API key replaces separate Google, Reddit, and YouTube ingestion tools. 3 queries per topic = $0.015.