Hermes Diario Automation Search Pipeline

import requests
import json
import time
from pathlib import Path
from datetime import datetime

API_KEY = "your_scavio_api_key"

AGENT_LOGS = Path("agent_logs.jsonl")
KNOWLEDGE_BASE = Path("hermes_knowledge.json")

def find_failed_queries() -> list[str]:
    """Parse agent logs for queries that produced uncertain responses."""
    failed = []
    if not AGENT_LOGS.exists():
        return failed
    for line in AGENT_LOGS.read_text().strip().split("\n"):
        try:
            entry = json.loads(line)
            if entry.get("confidence", 1.0) < 0.5 or entry.get("admitted_uncertainty"):
                failed.append(entry.get("query", ""))
        except json.JSONDecodeError:
            continue
    return [q for q in failed if q][-20]  # Last 20 failures

def research_query(query: str) -> dict:
    """Search multiple platforms for comprehensive coverage."""
    findings = {"query": query, "researched_at": datetime.utcnow().isoformat(), "sources": []}

    for platform in ["google", "reddit"]:
        try:
            res = requests.post(
                "https://api.scavio.dev/api/v1/search",
                headers={"x-api-key": API_KEY},
                json={"platform": platform, "query": query},
                timeout=10,
            )
            res.raise_for_status()
            results = res.json().get("organic", [])[:3]
            for r in results:
                findings["sources"].append({
                    "platform": platform,
                    "title": r.get("title", ""),
                    "snippet": r.get("snippet", ""),
                    "link": r.get("link", ""),
                })
        except Exception:
            continue

    return findings

def run():
    failed_queries = find_failed_queries()
    if not failed_queries:
        print("No failed queries to research")
        return

    # Load existing knowledge base
    knowledge = json.loads(KNOWLEDGE_BASE.read_text()) if KNOWLEDGE_BASE.exists() else {}

    researched = 0
    for query in failed_queries:
        findings = research_query(query)
        if findings["sources"]:
            knowledge[query] = findings
            researched += 1
        time.sleep(0.5)

    KNOWLEDGE_BASE.write_text(json.dumps(knowledge, indent=2))
    print(f"Researched {researched}/{len(failed_queries)} failed queries")
    print(f"Knowledge base now has {len(knowledge)} entries")

if __name__ == "__main__":
    run()

const API_KEY = "your_scavio_api_key";

async function findFailedQueries() {
  const fs = await import("fs/promises");
  let content;
  try { content = await fs.readFile("agent_logs.jsonl", "utf8"); } catch { return []; }
  const failed = [];
  for (const line of content.trim().split("\n")) {
    try {
      const entry = JSON.parse(line);
      if ((entry.confidence ?? 1) < 0.5 || entry.admitted_uncertainty) {
        if (entry.query) failed.push(entry.query);
      }
    } catch {}
  }
  return failed.slice(-20);
}

async function researchQuery(query) {
  const findings = { query, researchedAt: new Date().toISOString(), sources: [] };
  for (const platform of ["google", "reddit"]) {
    try {
      const res = await fetch("https://api.scavio.dev/api/v1/search", {
        method: "POST",
        headers: { "x-api-key": API_KEY, "content-type": "application/json" },
        body: JSON.stringify({ platform, query }),
      });
      if (!res.ok) continue;
      const results = ((await res.json()).organic ?? []).slice(0, 3);
      for (const r of results) {
        findings.sources.push({ platform, title: r.title ?? "", snippet: r.snippet ?? "", link: r.link ?? "" });
      }
    } catch {}
  }
  return findings;
}

async function run() {
  const fs = await import("fs/promises");
  const failedQueries = await findFailedQueries();
  if (!failedQueries.length) { console.log("No failed queries to research"); return; }

  let knowledge = {};
  try { knowledge = JSON.parse(await fs.readFile("hermes_knowledge.json", "utf8")); } catch {}

  let researched = 0;
  for (const query of failedQueries) {
    const findings = await researchQuery(query);
    if (findings.sources.length) {
      knowledge[query] = findings;
      researched++;
    }
    await new Promise((r) => setTimeout(r, 500));
  }

  await fs.writeFile("hermes_knowledge.json", JSON.stringify(knowledge, null, 2));
  console.log(`Researched ${researched}/${failedQueries.length} failed queries`);
  console.log(`Knowledge base: ${Object.keys(knowledge).length} entries`);
}

run();