How to Get Reddit Data Into an LLM: Search, Full Comment Trees and a Compact Prompt

curl -s https://api.scavio.dev/api/v1/reddit/search \
  -H "Authorization: Bearer $SCAVIO_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"query": "best embedding model for RAG"}'

{
  "post_id": "t3_1v2ebaq",
  "title": "Best embedding model for indexing ~17,000 scientific PDFs for a RAG system in 2026?",
  "text": "Hi everyone,\n\nI'm building a production RAG system focused on **scientific research papers** (desalination, chemistry, membranes, engineering). ...",
  "url": "https://www.reddit.com/r/Rag/comments/1v2ebaq/best_embedding_model_for_indexing_17000/",
  "subreddit": "Rag",
  "author": "Gintoki55",
  "score": 82,
  "upvote_ratio": 0.9880952380952381,
  "num_comments": 60,
  "created_at": "2026-07-21T10:16:56.369000+0000",
  "is_nsfw": false
}

curl -s https://api.scavio.dev/api/v1/reddit/post/comments \
  -H "Authorization: Bearer $SCAVIO_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"post_id": "t3_1v2ebaq"}'

[
  {
    "comment_id": "t1_oyvtf9y",
    "text": "Before re indexing everything, I'd also benchmark a simple Elasticsearch BM25 baseline on your own scientific queries. ...",
    "author": "MashoodKiyani05",
    "score": 10,
    "created_at": "2026-07-21T14:50:56.547000+0000",
    "depth": 0,
    "reply_cursor": null
  },
  {
    "comment_id": "t1_oyx76zw",
    "text": "**Could you elaborate on that?**",
    "author": "Gintoki55",
    "score": 1,
    "created_at": "2026-07-21T18:22:22.702000+0000",
    "depth": 1,
    "reply_cursor": null
  },
  {
    "comment_id": "t1_oyxefvj",
    "text": "I think they mean just using Elasticsearch - BM25 is the standard in most cases for inverted indexing. ...",
    "author": "5d10_shades_of_grey",
    "score": 3,
    "created_at": "2026-07-21T18:53:26.977000+0000",
    "depth": 2,
    "reply_cursor": null
  }
]

curl -s https://api.scavio.dev/api/v1/reddit/post/comments/replies \
  -H "Authorization: Bearer $SCAVIO_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"post_id": "t3_1v2ebaq", "cursor": "commenttree:07d44405,::1:d1:t1_oyumg52"}'

{
  "replies": [
    {
      "comment_id": "t1_oyx7jet",
      "text": "**Thanks for** **advise** **!**",
      "author": "Gintoki55",
      "score": 2,
      "created_at": "2026-07-21T18:23:52.174000+0000",
      "depth": 1,
      "reply_cursor": null
    }
  ],
  "next_cursor": null,
  "has_more": false
}

import requests

API = "https://api.scavio.dev/api/v1/reddit"
HEADERS = {"Authorization": "Bearer sk_live_your_key"}

def call(path, **body):
    r = requests.post(f"{API}/{path}", headers=HEADERS, json=body, timeout=60)
    r.raise_for_status()
    return r.json()["data"]

def fetch_tree(post_id):
    comments, cursor = [], None
    while True:
        page = call("post/comments", post_id=post_id, **({"cursor": cursor} if cursor else {}))
        comments += page["comments"]
        cursor = page["next_cursor"]
        if not page["has_more"] or not cursor:
            break
    tree = []
    def add(c):
        tree.append(c)
        rc = c.get("reply_cursor")
        while rc:  # collapsed branch: expand it under this comment
            page = call("post/comments/replies", post_id=post_id, cursor=rc)
            for r in page["replies"]:
                add(r)
            rc = page["next_cursor"] if page["has_more"] else None
    for c in comments:
        add(c)
    return tree

def to_text(post, tree):
    lines = [f"# {post['title']} (r/{post['subreddit']}, score {post['score']}, {post['num_comments']} comments)"]
    if post.get("text"):
        lines.append(post["text"])
    for c in tree:
        text = (c.get("text") or "").strip()
        if text in ("", "[deleted]", "[removed]"):
            continue
        lines.append("  " * c["depth"] + f"- [{c['score']}] " + text.replace("\n", " "))
    return "\n".join(lines)

from openai import OpenAI

posts = call("search", query="best embedding model for RAG")["results"][:3]
context = "\n\n".join(to_text(p, fetch_tree(p["post_id"])) for p in posts)

client = OpenAI()
answer = client.responses.create(
    model="gpt-5.5",
    input="Using only these Reddit threads, which embedding models do practitioners "
          "recommend for RAG over technical documents, and why? Cite the subreddit.\n\n" + context,
)
print(answer.output_text)