Extrair Conteúdo como Markdown

Converta qualquer página web em Markdown limpo usando a API de conteúdo CaptureKit — remova anúncios e código HTML desnecessário para obter o conteúdo puro do artigo ou documentação.

Visão geral

O endpoint de conteúdo CaptureKit obtém uma página web, remove navegação, anúncios e elementos de layout, devolvendo depois o corpo principal como Markdown limpo. Use-o para construir datasets RAG, indexar documentação ou arquivar artigos.

Pré-requisitos

Uma chave de API CaptureKit — obtenha uma em app.capturekit.dev
Instale as dependências para o seu idioma:

pip install requests

Sem dependências adicionais — utiliza a API fetch nativa (Node 18+).

Extensão curl ativada (ativa por padrão).

Sem dependências adicionais — utiliza net/http (Go 1.18+).

Sem dependências adicionais — utiliza java.net.http (Java 11+).

Sem dependências adicionais — utiliza System.Net.Http (.NET 6+).

# Cargo.toml
[dependencies]
reqwest = { version = "0.12", features = ["json"] }
tokio = { version = "1", features = ["full"] }
serde_json = "1"

Passos

Obter o conteúdo da página

Chame GET /v1/content com o parâmetro url.

import requests

API_KEY  = "YOUR_API_KEY"
response = requests.get(
    "https://api.capturekit.dev/v1/content",
    headers={"x-api-key": API_KEY},
    params={"url": "https://stripe.com/docs/payments"},
)
data = response.json()
print(data)

const API_KEY = "YOUR_API_KEY";

const params   = new URLSearchParams({ url: "https://stripe.com/docs/payments" });
const response = await fetch(`https://api.capturekit.dev/v1/content?${params}`, {
  headers: { "x-api-key": API_KEY },
});
const data = await response.json();
console.log(data);

<?php
$apiKey = "YOUR_API_KEY";
$params = http_build_query(["url" => "https://stripe.com/docs/payments"]);

$ch = curl_init("https://api.capturekit.dev/v1/content?{$params}");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, ["x-api-key: {$apiKey}"]);
$data = json_decode(curl_exec($ch), true);
curl_close($ch);
print_r($data);

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "net/url"
)

func main() {
    params := url.Values{"url": {"https://stripe.com/docs/payments"}}
    req, _ := http.NewRequest("GET", "https://api.capturekit.dev/v1/content?"+params.Encode(), nil)
    req.Header.Set("x-api-key", "YOUR_API_KEY")

    resp, _ := http.DefaultClient.Do(req)
    defer resp.Body.Close()
    body, _ := io.ReadAll(resp.Body)

    var data map[string]any
    json.Unmarshal(body, &data)
    fmt.Println(data)
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.*;
import java.nio.charset.StandardCharsets;

var client  = HttpClient.newHttpClient();
var target  = URLEncoder.encode("https://stripe.com/docs/payments", StandardCharsets.UTF_8);
var request = HttpRequest.newBuilder()
    .uri(URI.create("https://api.capturekit.dev/v1/content?url=" + target))
    .header("x-api-key", "YOUR_API_KEY").GET().build();

var response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

using System.Net.Http;

using var client = new HttpClient();
client.DefaultRequestHeaders.Add("x-api-key", "YOUR_API_KEY");

var target = Uri.EscapeDataString("https://stripe.com/docs/payments");
var body   = await client.GetStringAsync($"https://api.capturekit.dev/v1/content?url={target}");
Console.WriteLine(body);

#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
    let data = reqwest::Client::new()
        .get("https://api.capturekit.dev/v1/content")
        .header("x-api-key", "YOUR_API_KEY")
        .query(&[("url", "https://stripe.com/docs/payments")])
        .send().await?.json::<serde_json::Value>().await?;

    println!("{:#?}", data);
    Ok(())
}

Aceder ao conteúdo Markdown

A resposta inclui markdown, title, description, author, published_at e word_count.

print(f"Título        : {data.get('title')}")
print(f"Autor         : {data.get('author', 'N/D')}")
print(f"Contagem pals.: {data.get('word_count')} palavras")
print()
print("--- Pré-visualização Markdown ---")
markdown = data.get("markdown", "")
print(markdown[:1000])

console.log(`Título        : ${data.title}`);
console.log(`Autor         : ${data.author ?? "N/D"}`);
console.log(`Contagem pals.: ${data.word_count} palavras\n`);
console.log("--- Pré-visualização Markdown ---");
console.log(data.markdown?.slice(0, 1000));

echo "Título        : {$data['title']}\n";
echo "Autor         : " . ($data["author"] ?? "N/D") . "\n";
echo "Contagem pals.: {$data['word_count']} palavras\n\n";
echo "--- Pré-visualização Markdown ---\n";
echo substr($data["markdown"] ?? "", 0, 1000) . "\n";

fmt.Printf("Título        : %v\nAutor         : %v\nContagem pals.: %v palavras\n\n",
    data["title"], data["author"], data["word_count"])
markdown := data["markdown"].(string)
if len(markdown) > 1000 { markdown = markdown[:1000] }
fmt.Println("--- Pré-visualização Markdown ---\n" + markdown)

import org.json.*;

var d        = new JSONObject(response.body());
var markdown = d.getString("markdown");
System.out.printf("Título        : %s%nAutor         : %s%nContagem pals.: %d palavras%n%n",
    d.getString("title"), d.optString("author", "N/D"), d.getInt("word_count"));
System.out.println("--- Pré-visualização Markdown ---");
System.out.println(markdown.substring(0, Math.min(1000, markdown.length())));

using System.Text.Json;

var d        = JsonDocument.Parse(body).RootElement;
var markdown = d.GetProperty("markdown").GetString() ?? "";
Console.WriteLine($"Título        : {d.GetProperty("title")}");
Console.WriteLine($"Autor         : {(d.TryGetProperty("author", out var a) ? a : (object)"N/D")}");
Console.WriteLine($"Contagem pals.: {d.GetProperty("word_count")} palavras\n");
Console.WriteLine("--- Pré-visualização Markdown ---");
Console.WriteLine(markdown[..Math.Min(1000, markdown.Length)]);

let markdown = data["markdown"].as_str().unwrap_or("");
println!("Título        : {}", data["title"].as_str().unwrap_or(""));
println!("Autor         : {}", data["author"].as_str().unwrap_or("N/D"));
println!("Contagem pals.: {} palavras\n", data["word_count"]);
println!("--- Pré-visualização Markdown ---");
println!("{}", &markdown[..1000.min(markdown.len())]);

Rastrear e arquivar uma lista de páginas de documentação

Obtenha e guarde múltiplas páginas como ficheiros Markdown para pesquisa offline ou ingestão RAG.

import os, time, requests

API_KEY = "YOUR_API_KEY"
os.makedirs("arquivo_docs", exist_ok=True)

PAGES = [
    "https://stripe.com/docs/payments",
    "https://stripe.com/docs/billing",
    "https://stripe.com/docs/connect",
]

for page_url in PAGES:
    r    = requests.get("https://api.capturekit.dev/v1/content",
        headers={"x-api-key": API_KEY}, params={"url": page_url})
    data = r.json()
    slug = page_url.rstrip("/").split("/")[-1]
    path = f"arquivo_docs/{slug}.md"

    with open(path, "w") as f:
        f.write(f"# {data.get('title', slug)}\n\n")
        f.write(data.get("markdown", ""))

    print(f"Guardado: {path} ({data.get('word_count', 0)} palavras)")
    time.sleep(1)

print("Concluído!")

import { mkdirSync, writeFileSync } from "fs";

const API_KEY = "YOUR_API_KEY";
mkdirSync("arquivo_docs", { recursive: true });

const PAGES = [
  "https://stripe.com/docs/payments",
  "https://stripe.com/docs/billing",
  "https://stripe.com/docs/connect",
];

for (const pageUrl of PAGES) {
  const params = new URLSearchParams({ url: pageUrl });
  const res    = await fetch(`https://api.capturekit.dev/v1/content?${params}`, {
    headers: { "x-api-key": API_KEY },
  });
  const data = await res.json();
  const slug = pageUrl.replace(/\/$/, "").split("/").at(-1)!;
  writeFileSync(`arquivo_docs/${slug}.md`, `# ${data.title ?? slug}\n\n${data.markdown ?? ""}`);
  console.log(`Guardado: arquivo_docs/${slug}.md (${data.word_count ?? 0} palavras)`);
  await new Promise(r => setTimeout(r, 1000));
}
console.log("Concluído!");

<?php
$apiKey = "YOUR_API_KEY";
@mkdir("arquivo_docs");

$pages = [
    "https://stripe.com/docs/payments",
    "https://stripe.com/docs/billing",
    "https://stripe.com/docs/connect",
];

foreach ($pages as $pageUrl) {
    $params = http_build_query(["url" => $pageUrl]);
    $ch = curl_init("https://api.capturekit.dev/v1/content?{$params}");
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_HTTPHEADER, ["x-api-key: {$apiKey}"]);
    $data = json_decode(curl_exec($ch), true);
    curl_close($ch);

    $slug = basename(rtrim($pageUrl, "/"));
    $path = "arquivo_docs/{$slug}.md";
    file_put_contents($path, "# {$data['title']}\n\n{$data['markdown']}");
    echo "Guardado: {$path} ({$data['word_count']} palavras)\n";
    sleep(1);
}
echo "Concluído!\n";

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "net/url"
    "os"
    "path/filepath"
    "strings"
    "time"
)

func main() {
    os.MkdirAll("arquivo_docs", 0755)
    apiKey := "YOUR_API_KEY"
    pages  := []string{
        "https://stripe.com/docs/payments",
        "https://stripe.com/docs/billing",
        "https://stripe.com/docs/connect",
    }

    for _, pageURL := range pages {
        params := url.Values{"url": {pageURL}}
        req, _ := http.NewRequest("GET", "https://api.capturekit.dev/v1/content?"+params.Encode(), nil)
        req.Header.Set("x-api-key", apiKey)
        resp, _ := http.DefaultClient.Do(req)
        body, _ := io.ReadAll(resp.Body)
        resp.Body.Close()

        var data map[string]any
        json.Unmarshal(body, &data)

        parts := strings.Split(strings.TrimRight(pageURL, "/"), "/")
        slug  := parts[len(parts)-1]
        path  := filepath.Join("arquivo_docs", slug+".md")
        content := fmt.Sprintf("# %v\n\n%v", data["title"], data["markdown"])
        os.WriteFile(path, []byte(content), 0644)
        fmt.Printf("Guardado: %s (%.0f palavras)\n", path, data["word_count"])
        time.Sleep(time.Second)
    }
    fmt.Println("Concluído!")
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.List;
import org.json.*;

public class Main {
    public static void main(String[] args) throws Exception {
        var client = HttpClient.newHttpClient();
        var apiKey = "YOUR_API_KEY";
        Files.createDirectories(Path.of("arquivo_docs"));

        var pages = List.of(
            "https://stripe.com/docs/payments",
            "https://stripe.com/docs/billing",
            "https://stripe.com/docs/connect");

        for (var pageUrl : pages) {
            var encoded = URLEncoder.encode(pageUrl, StandardCharsets.UTF_8);
            var req     = HttpRequest.newBuilder()
                .uri(URI.create("https://api.capturekit.dev/v1/content?url=" + encoded))
                .header("x-api-key", apiKey).GET().build();
            var resp = client.send(req, HttpResponse.BodyHandlers.ofString());
            var data = new JSONObject(resp.body());

            var slug    = pageUrl.replaceAll("/$","").replaceAll(".*/","");
            var content = "# " + data.getString("title") + "\n\n" + data.getString("markdown");
            Files.writeString(Path.of("arquivo_docs/" + slug + ".md"), content);
            System.out.printf("Guardado: arquivo_docs/%s.md (%d palavras)%n", slug, data.getInt("word_count"));
            Thread.sleep(1000);
        }
        System.out.println("Concluído!");
    }
}

using System.Net.Http;
using System.Text.Json;

var apiKey = "YOUR_API_KEY";
using var client = new HttpClient();
client.DefaultRequestHeaders.Add("x-api-key", apiKey);
Directory.CreateDirectory("arquivo_docs");

var pages = new[]
{
    "https://stripe.com/docs/payments",
    "https://stripe.com/docs/billing",
    "https://stripe.com/docs/connect",
};

foreach (var pageUrl in pages)
{
    var encoded = Uri.EscapeDataString(pageUrl);
    var body    = await client.GetStringAsync($"https://api.capturekit.dev/v1/content?url={encoded}");
    var data    = JsonDocument.Parse(body).RootElement;
    var slug    = pageUrl.TrimEnd('/').Split('/').Last();
    var content = $"# {data.GetProperty("title")}\n\n{data.GetProperty("markdown")}";
    File.WriteAllText($"arquivo_docs/{slug}.md", content);
    Console.WriteLine($"Guardado: arquivo_docs/{slug}.md ({data.GetProperty("word_count")} palavras)");
    await Task.Delay(1000);
}
Console.WriteLine("Concluído!");

use reqwest::Client;
use serde_json::Value;
use std::{fs, path::Path, time::Duration};
use tokio::time::sleep;

#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
    let client  = Client::new();
    let api_key = "YOUR_API_KEY";
    let pages   = ["https://stripe.com/docs/payments", "https://stripe.com/docs/billing", "https://stripe.com/docs/connect"];
    fs::create_dir_all("arquivo_docs").unwrap();

    for page_url in &pages {
        let data = client.get("https://api.capturekit.dev/v1/content")
            .header("x-api-key", api_key)
            .query(&[("url", page_url)])
            .send().await?.json::<Value>().await?;

        let slug     = page_url.trim_end_matches('/').split('/').last().unwrap_or("pagina");
        let path     = format!("arquivo_docs/{}.md", slug);
        let content  = format!("# {}\n\n{}", data["title"].as_str().unwrap_or(""), data["markdown"].as_str().unwrap_or(""));
        fs::write(&path, content).unwrap();
        println!("Guardado: {} ({} palavras)", path, data["word_count"]);
        sleep(Duration::from_secs(1)).await;
    }
    println!("Concluído!");
    Ok(())
}

O Markdown extraído é ideal como blocos de contexto para uma pipeline RAG (Retrieval-Augmented Generation). Divida por cabeçalhos e incorpore no seu armazenamento vetorial preferido para pesquisa semântica sobre a documentação de qualquer site.