Deep Research Daily Pipeline

import requests
import json
from pathlib import Path
from datetime import datetime

API_KEY = "your_scavio_api_key"
BASE = "https://api.scavio.dev/api/v1"

TOPICS = ["AI agent search tools 2026", "SERP API pricing changes", "MCP server adoption"]

def search_platform(query: str, platform: str) -> list[dict]:
    res = requests.post(
        f"{BASE}/search",
        headers={"x-api-key": API_KEY},
        json={"platform": platform, "query": query},
        timeout=15,
    )
    res.raise_for_status()
    return res.json().get("organic", [])

def extract_content(url: str) -> dict:
    res = requests.post(
        f"{BASE}/extract",
        headers={"x-api-key": API_KEY},
        json={"url": url},
        timeout=30,
    )
    res.raise_for_status()
    return res.json()

def research_topic(topic: str) -> dict:
    google_results = search_platform(topic, "google")
    reddit_results = search_platform(topic, "reddit")
    youtube_results = search_platform(topic, "youtube")

    # Extract top 3 Google results
    extracted = []
    for result in google_results[:3]:
        url = result.get("link", "")
        if url:
            try:
                content = extract_content(url)
                extracted.append({
                    "url": url,
                    "title": result.get("title", ""),
                    "content_preview": content.get("text", "")[:500],
                })
            except Exception:
                pass

    return {
        "topic": topic,
        "google_count": len(google_results),
        "reddit_count": len(reddit_results),
        "youtube_count": len(youtube_results),
        "extracted_pages": len(extracted),
        "top_reddit": [{"title": r.get("title", ""), "score": r.get("score", 0)} for r in reddit_results[:5]],
        "top_youtube": [{"title": r.get("title", ""), "views": r.get("views", 0)} for r in youtube_results[:5]],
        "extracted": extracted,
    }

def run():
    date = datetime.utcnow().strftime("%Y-%m-%d")
    briefs = [research_topic(t) for t in TOPICS]
    total_credits = sum(3 + b["extracted_pages"] for b in briefs)  # 3 searches + extractions per topic
    report = {"date": date, "topics": len(TOPICS), "credits_used": total_credits, "briefs": briefs}
    Path(f"research_{date}.json").write_text(json.dumps(report, indent=2))
    print(f"Research complete: {len(TOPICS)} topics, {total_credits} credits")
    for brief in briefs:
        print(f"  {brief['topic']}: {brief['google_count']}G {brief['reddit_count']}R {brief['youtube_count']}Y {brief['extracted_pages']}E")

if __name__ == "__main__":
    run()

const API_KEY = "your_scavio_api_key";
const BASE = "https://api.scavio.dev/api/v1";
const TOPICS = ["AI agent search tools 2026", "SERP API pricing changes", "MCP server adoption"];

async function search(query, platform) {
  const res = await fetch(`${BASE}/search`, {
    method: "POST",
    headers: { "x-api-key": API_KEY, "content-type": "application/json" },
    body: JSON.stringify({ platform, query }),
  });
  if (!res.ok) throw new Error(`scavio ${res.status}`);
  return (await res.json()).organic ?? [];
}

async function extract(url) {
  const res = await fetch(`${BASE}/extract`, {
    method: "POST",
    headers: { "x-api-key": API_KEY, "content-type": "application/json" },
    body: JSON.stringify({ url }),
  });
  if (!res.ok) return null;
  return res.json();
}

async function run() {
  const fs = await import("fs/promises");
  const briefs = [];
  for (const topic of TOPICS) {
    const [google, reddit, youtube] = await Promise.all([
      search(topic, "google"), search(topic, "reddit"), search(topic, "youtube"),
    ]);
    const extracted = [];
    for (const r of google.slice(0, 3)) {
      if (r.link) { const c = await extract(r.link); if (c) extracted.push({ url: r.link, title: r.title ?? "", preview: (c.text ?? "").slice(0, 500) }); }
    }
    briefs.push({ topic, google: google.length, reddit: reddit.length, youtube: youtube.length, extracted: extracted.length });
  }
  const date = new Date().toISOString().slice(0, 10);
  await fs.writeFile(`research_${date}.json`, JSON.stringify(briefs, null, 2));
  for (const b of briefs) console.log(`  ${b.topic}: ${b.google}G ${b.reddit}R ${b.youtube}Y ${b.extracted}E`);
}

run();