Scraper les résultats Google

Construisez un script de surveillance SERP qui suit les positions de mots-clés et extrait les résultats organiques avec l'API Autom.

Vue d'ensemble

Dans ce guide, vous allez construire un script qui interroge Google Search et extrait des résultats organiques structurés — positions, titres, URLs et extraits — pour n'importe quel mot-clé. Un cas d'usage typique est le suivi de positions : exécutez ce script régulièrement pour surveiller où vos pages apparaissent pour vos mots-clés cibles.

Le endpoint renvoie jusqu'à 10 résultats organiques par page et prend en charge la pagination, le ciblage par pays (gl) et par langue (hl).

Prérequis

Une clé API Autom — obtenez-en une sur app.autom.dev
Installez les dépendances pour votre langage :

pip install requests

Aucune dépendance supplémentaire — utilise l'API native fetch (Node 18+).

Extension curl activée (par défaut dans la plupart des installations PHP).

Aucune dépendance supplémentaire — utilise la bibliothèque standard net/http (Go 1.18+).

Aucune dépendance supplémentaire — utilise java.net.http (Java 11+).

Aucune dépendance supplémentaire — utilise System.Net.Http (.NET 6+).

# Cargo.toml
[dependencies]
reqwest = { version = "0.12", features = ["json"] }
tokio = { version = "1", features = ["full"] }
serde_json = "1"

Étapes

Effectuer votre première requête de recherche

Appelez GET /v1/google/search avec le paramètre q et votre clé API dans l'en-tête x-api-key.

import requests

API_KEY = "YOUR_API_KEY"

response = requests.get(
    "https://api.autom.dev/v1/google/search",
    headers={"x-api-key": API_KEY},
    params={"q": "best python web scraping libraries", "gl": "us", "hl": "en"},
)

data = response.json()
print(data)

const API_KEY = "YOUR_API_KEY";

const params = new URLSearchParams({
  q: "best python web scraping libraries",
  gl: "us",
  hl: "en",
});

const response = await fetch(
  `https://api.autom.dev/v1/google/search?${params}`,
  { headers: { "x-api-key": API_KEY } },
);

const data = await response.json();
console.log(data);

<?php
$apiKey = "YOUR_API_KEY";

$params = http_build_query([
  "q"  => "best python web scraping libraries",
  "gl" => "us",
  "hl" => "en",
]);

$ch = curl_init("https://api.autom.dev/v1/google/search?{$params}");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, ["x-api-key: {$apiKey}"]);

$data = json_decode(curl_exec($ch), true);
curl_close($ch);
print_r($data);

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "net/url"
)

func main() {
    apiKey := "YOUR_API_KEY"

    params := url.Values{}
    params.Set("q",  "best python web scraping libraries")
    params.Set("gl", "us")
    params.Set("hl", "en")

    req, _ := http.NewRequest("GET",
        "https://api.autom.dev/v1/google/search?"+params.Encode(), nil)
    req.Header.Set("x-api-key", apiKey)

    resp, _ := http.DefaultClient.Do(req)
    defer resp.Body.Close()

    body, _ := io.ReadAll(resp.Body)
    var data map[string]any
    json.Unmarshal(body, &data)
    fmt.Println(data)
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) throws Exception {
        var apiKey = "YOUR_API_KEY";
        var q      = URLEncoder.encode("best python web scraping libraries", StandardCharsets.UTF_8);
        var url    = "https://api.autom.dev/v1/google/search?q=" + q + "&gl=us&hl=en";

        var client  = HttpClient.newHttpClient();
        var request = HttpRequest.newBuilder()
            .uri(URI.create(url))
            .header("x-api-key", apiKey)
            .GET()
            .build();

        var response = client.send(request, HttpResponse.BodyHandlers.ofString());
        System.out.println(response.body());
    }
}

using System.Net.Http;

var apiKey = "YOUR_API_KEY";
using var client = new HttpClient();
client.DefaultRequestHeaders.Add("x-api-key", apiKey);

var url      = "https://api.autom.dev/v1/google/search?q=best+python+web+scraping+libraries&gl=us&hl=en";
var response = await client.GetAsync(url);
var body     = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);

use reqwest::header;

#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
    let client = reqwest::Client::new();

    let response = client
        .get("https://api.autom.dev/v1/google/search")
        .header("x-api-key", "YOUR_API_KEY")
        .query(&[("q", "best python web scraping libraries"), ("gl", "us"), ("hl", "en")])
        .send()
        .await?
        .json::<serde_json::Value>()
        .await?;

    println!("{:#?}", response);
    Ok(())
}

Analyser les résultats organiques

La réponse contient un tableau organic_results. Chaque entrée possède position, title, link, snippet, domain et source.

for result in data["organic_results"]:
    print(f"[{result['position']}] {result['title']}")
    print(f"    URL: {result['link']}")
    print(f"    {result.get('snippet', '')}")
    print()

for (const result of data.organic_results) {
  console.log(`[${result.position}] ${result.title}`);
  console.log(`    URL: ${result.link}`);
  console.log(`    ${result.snippet ?? ""}`);
  console.log();
}

foreach ($data["organic_results"] as $result) {
    echo "[{$result['position']}] {$result['title']}\n";
    echo "    URL: {$result['link']}\n";
    echo "    " . ($result['snippet'] ?? "") . "\n\n";
}

results := data["organic_results"].([]any)
for _, r := range results {
    item := r.(map[string]any)
    fmt.Printf("[%.0f] %s\n", item["position"], item["title"])
    fmt.Printf("    URL: %s\n", item["link"])
    fmt.Printf("    %s\n\n", item["snippet"])
}

import org.json.JSONArray;
import org.json.JSONObject;

var json    = new JSONObject(response.body());
var results = json.getJSONArray("organic_results");

for (int i = 0; i < results.length(); i++) {
    var r = results.getJSONObject(i);
    System.out.printf("[%d] %s%n", r.getInt("position"), r.getString("title"));
    System.out.printf("    URL: %s%n%n", r.getString("link"));
}

using System.Text.Json;

var json    = JsonDocument.Parse(body);
var results = json.RootElement.GetProperty("organic_results").EnumerateArray();

foreach (var result in results)
{
    Console.WriteLine($"[{result.GetProperty("position")}] {result.GetProperty("title")}");
    Console.WriteLine($"    URL: {result.GetProperty("link")}");
    Console.WriteLine();
}

if let Some(results) = response["organic_results"].as_array() {
    for result in results {
        println!(
            "[{}] {}",
            result["position"],
            result["title"].as_str().unwrap_or("")
        );
        println!("    URL: {}", result["link"].as_str().unwrap_or(""));
        println!("    {}", result["snippet"].as_str().unwrap_or(""));
        println!();
    }
}

Gérer la pagination

Utilisez le paramètre page pour récupérer les pages suivantes. La réponse inclut un objet pagination avec has_next_page et le numéro de next page.

import requests

API_KEY = "YOUR_API_KEY"
QUERY   = "best python web scraping libraries"

def fetch_all_results(query: str, max_pages: int = 3) -> list:
    all_results = []
    page = 1

    while page <= max_pages:
        response = requests.get(
            "https://api.autom.dev/v1/google/search",
            headers={"x-api-key": API_KEY},
            params={"q": query, "gl": "us", "hl": "en", "page": page},
        )
        data = response.json()
        all_results.extend(data.get("organic_results", []))

        if not data.get("pagination", {}).get("has_next_page"):
            break
        page += 1

    return all_results

results = fetch_all_results(QUERY)
for r in results:
    print(f"[{r['position']}] {r['title']} — {r['link']}")

const API_KEY = "YOUR_API_KEY";

async function fetchAllResults(query: string, maxPages = 3) {
  const allResults: any[] = [];
  let page = 1;

  while (page <= maxPages) {
    const params = new URLSearchParams({ q: query, gl: "us", hl: "en", page: String(page) });
    const res = await fetch(`https://api.autom.dev/v1/google/search?${params}`, {
      headers: { "x-api-key": API_KEY },
    });
    const data = await res.json();
    allResults.push(...(data.organic_results ?? []));

    if (!data.pagination?.has_next_page) break;
    page++;
  }

  return allResults;
}

const results = await fetchAllResults("best python web scraping libraries");
for (const r of results) {
  console.log(`[${r.position}] ${r.title} — ${r.link}`);
}

<?php
function fetchAllResults(string $apiKey, string $query, int $maxPages = 3): array {
    $allResults = [];
    $page = 1;

    while ($page <= $maxPages) {
        $params = http_build_query(["q" => $query, "gl" => "us", "hl" => "en", "page" => $page]);
        $ch = curl_init("https://api.autom.dev/v1/google/search?{$params}");
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_HTTPHEADER, ["x-api-key: {$apiKey}"]);
        $data = json_decode(curl_exec($ch), true);
        curl_close($ch);

        $allResults = array_merge($allResults, $data["organic_results"] ?? []);
        if (!($data["pagination"]["has_next_page"] ?? false)) break;
        $page++;
    }
    return $allResults;
}

$results = fetchAllResults("YOUR_API_KEY", "best python web scraping libraries");
foreach ($results as $r) {
    echo "[{$r['position']}] {$r['title']} — {$r['link']}\n";
}

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "net/url"
    "strconv"
)

func fetchAllResults(apiKey, query string, maxPages int) []map[string]any {
    var all []map[string]any

    for page := 1; page <= maxPages; page++ {
        params := url.Values{"q": {query}, "gl": {"us"}, "hl": {"en"}, "page": {strconv.Itoa(page)}}
        req, _ := http.NewRequest("GET", "https://api.autom.dev/v1/google/search?"+params.Encode(), nil)
        req.Header.Set("x-api-key", apiKey)

        resp, _ := http.DefaultClient.Do(req)
        body, _ := io.ReadAll(resp.Body)
        resp.Body.Close()

        var data map[string]any
        json.Unmarshal(body, &data)

        if items, ok := data["organic_results"].([]any); ok {
            for _, item := range items {
                all = append(all, item.(map[string]any))
            }
        }

        pagination, _ := data["pagination"].(map[string]any)
        if pagination["has_next_page"] != true {
            break
        }
    }
    return all
}

func main() {
    results := fetchAllResults("YOUR_API_KEY", "best python web scraping libraries", 3)
    for _, r := range results {
        fmt.Printf("[%.0f] %s — %s\n", r["position"], r["title"], r["link"])
    }
}

import java.net.URI;
import java.net.URLEncoder;
import java.net.http.*;
import java.nio.charset.StandardCharsets;
import java.util.*;
import org.json.*;

public class Main {
    static HttpClient client = HttpClient.newHttpClient();
    static String API_KEY   = "YOUR_API_KEY";

    static List<JSONObject> fetchAllResults(String query, int maxPages) throws Exception {
        var all  = new ArrayList<JSONObject>();
        var q    = URLEncoder.encode(query, StandardCharsets.UTF_8);

        for (int page = 1; page <= maxPages; page++) {
            var url     = "https://api.autom.dev/v1/google/search?q=" + q + "&gl=us&hl=en&page=" + page;
            var request = HttpRequest.newBuilder().uri(URI.create(url))
                .header("x-api-key", API_KEY).GET().build();
            var resp    = client.send(request, HttpResponse.BodyHandlers.ofString());
            var json    = new JSONObject(resp.body());
            var results = json.optJSONArray("organic_results");

            if (results != null) {
                for (int i = 0; i < results.length(); i++) all.add(results.getJSONObject(i));
            }
            if (!json.optJSONObject("pagination").optBoolean("has_next_page")) break;
        }
        return all;
    }

    public static void main(String[] args) throws Exception {
        for (var r : fetchAllResults("best python web scraping libraries", 3)) {
            System.out.printf("[%d] %s — %s%n", r.getInt("position"), r.getString("title"), r.getString("link"));
        }
    }
}

using System.Net.Http;
using System.Text.Json;

var apiKey = "YOUR_API_KEY";
using var client = new HttpClient();
client.DefaultRequestHeaders.Add("x-api-key", apiKey);

var allResults = new List<JsonElement>();
int page = 1, maxPages = 3;

while (page <= maxPages)
{
    var url      = $"https://api.autom.dev/v1/google/search?q=best+python+web+scraping+libraries&gl=us&hl=en&page={page}";
    var response = await client.GetAsync(url);
    var body     = await response.Content.ReadAsStringAsync();
    var json     = JsonDocument.Parse(body).RootElement;

    foreach (var result in json.GetProperty("organic_results").EnumerateArray())
        allResults.Add(result);

    var hasNext = json.GetProperty("pagination").GetProperty("has_next_page").GetBoolean();
    if (!hasNext) break;
    page++;
}

foreach (var r in allResults)
    Console.WriteLine($"[{r.GetProperty("position")}] {r.GetProperty("title")} — {r.GetProperty("link")}");

use reqwest::Client;
use serde_json::Value;

async fn fetch_all_results(client: &Client, api_key: &str, query: &str, max_pages: u32) -> Vec<Value> {
    let mut all_results = Vec::new();
    let mut page = 1u32;

    while page <= max_pages {
        let resp = client
            .get("https://api.autom.dev/v1/google/search")
            .header("x-api-key", api_key)
            .query(&[("q", query), ("gl", "us"), ("hl", "en"), ("page", &page.to_string())])
            .send().await.unwrap()
            .json::<Value>().await.unwrap();

        if let Some(results) = resp["organic_results"].as_array() {
            all_results.extend(results.clone());
        }
        if !resp["pagination"]["has_next_page"].as_bool().unwrap_or(false) { break; }
        page += 1;
    }
    all_results
}

#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
    let client  = Client::new();
    let results = fetch_all_results(&client, "YOUR_API_KEY", "best python web scraping libraries", 3).await;

    for r in &results {
        println!("[{}] {} — {}", r["position"], r["title"].as_str().unwrap_or(""), r["link"].as_str().unwrap_or(""));
    }
    Ok(())
}

Utilisez le paramètre gl pour cibler un pays spécifique (ex. fr pour la France, de pour l'Allemagne) et hl pour la langue des résultats. Cela est essentiel pour un suivi de positions local précis.