Clean Markdown and HTML output for RAG and AI agents

import requests

API_KEY = "YOUR_API_KEY"
ENDPOINT = f"https://publisher.scrappey.com/api/v1?key={API_KEY}"

def fetch_markdown(url: str) -> str:
    payload = {
        "cmd": "request.get",
        "url": url,
        "markdownResponse": True,
    }
    res = requests.post(ENDPOINT, json=payload, timeout=180)
    res.raise_for_status()
    data = res.json()
    # Clean, LLM-ready Markdown for the requested page
    return data["solution"]["response"]

def chunk(text: str, size: int = 1200, overlap: int = 150):
    chunks, start = [], 0
    while start < len(text):
        end = start + size
        chunks.append(text[start:end])
        start = end - overlap
    return chunks

markdown = fetch_markdown("https://example.com")
for i, c in enumerate(chunk(markdown)):
    print(f"--- chunk {i} ({len(c)} chars) ---")
    print(c)
    # embed(c) -> vector_store.upsert(...)

const API_KEY = process.env.SCRAPPEY_API_KEY;
const ENDPOINT = `https://publisher.scrappey.com/api/v1?key=${API_KEY}`;

async function fetchMarkdown(url) {
  const res = await fetch(ENDPOINT, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      cmd: "request.get",
      url,
      markdownResponse: true,
    }),
  });

  const data = await res.json();
  return data.solution.response; // LLM-ready Markdown
}

// Use as a retrieval/agent tool
const markdown = await fetchMarkdown("https://example.com");
console.log(markdown);