Passing API Parameters as Headers

Along with the "traditional" means of passing API parameters, they can be passed as part of the request headers as well. For example: api_key, render, ultra_premium and instruction_set.

Please note that the 'x-sapi-' prefix is used on each header to avoid collisions with headers used by target sites. We support all standard parameters available with the API. The instruction_set parameter is specifically supported only through headers.

• API REQUEST (URL Parameters)

cURL

curl --request GET \
  --url 'https://api.scraperapi.com?api_key=API_KEY&render=true&url=https://example.com/'

Python

import requests

target_url = 'https://example.com/'
# Replace the value for api_key with your actual API Key.
api_key = 'API_KEY'

request_url = (
    f'https://api.scraperapi.com?'
    f'api_key={api_key}'
    f'&render=true'
    f'&url={target_url}'
)

response = requests.get(request_url)
print(response.text)

NodeJS

import request from 'node-fetch';

// Replace the value for api_key with your actual API Key.
const url = 'http://api.scraperapi.com/?api_key=API_KEY&render=true&url=https://example.com/';

request(url)
  .then(response => {
    console.log(response);
  })
  .catch(error => {
    console.error(error);
  });

PHP

<?php

// Replace the value for api_key with your actual API Key.
$url = "https://api.scraperapi.com?api_key=API_KEY&render=true&url=https://example.com";

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, false);

$response = curl_exec($ch);

if (curl_errno($ch)) {
    echo 'Curl error: ' . curl_error($ch);
} else {
    print_r($response);
}

curl_close($ch);

Ruby

require 'net/http'

# Replace the value for api_key with your actual API Key.
params = {
  api_key: "API_KEY",
  render: "true",
  url: "https://example.com/"
}

uri = URI('https://api.scraperapi.com/')
uri.query = URI.encode_www_form(params)

website_content = Net::HTTP.get(uri)
puts website_content

Java

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

public class Main {
    public static void main(String[] args) throws Exception {
        // Replace the value for api_key with your actual API Key.
        String apiKey = "API_KEY";
        String targetUrl = "https://example.com/";

        String scraperApiUrl =
                "https://api.scraperapi.com?api_key=" + apiKey
                + "&render=true"
                + "&url=" + targetUrl;

        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request = HttpRequest.newBuilder()
                .uri(URI.create(scraperApiUrl))
                .GET()
                .build();

        HttpResponse<String> response =
                client.send(request, HttpResponse.BodyHandlers.ofString());

        System.out.println(response.body());
    }
}

• API REQUEST (Headers Parameters)

cURL

 curl --request GET \
  --url 'https://api.scraperapi.com?url=https://www.example.com' \
  --header 'x-sapi-api_key: API_KEY' \
  --header 'x-sapi-render: true'

Python

import requests

url = "https://api.scraperapi.com"
params = {
    "url": "https://www.example.com"
}

headers = {
    # Replace the value for api_key with your actual API Key.
    "x-sapi-api_key": "API_KEY",
    "x-sapi-render": "true"
}

response = requests.get(url, params=params, headers=headers)
print(response.text)

NodeJS

import fetch from "node-fetch";

const url = new URL("https://api.scraperapi.com");
url.searchParams.append("url", "https://www.example.com");

const response = await fetch(url, {
  method: "GET",
  headers: {
    "x-sapi-api_key": "API_KEY",
    "x-sapi-render": "true",
  },
});

const body = await response.text();
console.log(body);

PHP

<?php
$url = "https://api.scraperapi.com?url=https://www.example.com";
$ch = curl_init();

$headers = array(
    // Replace the value for api_key with your actual API Key.
    'x-sapi-api_key: API_KEY',
    'x-sapi-render: true'
);

curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_HEADER, FALSE);

$response = curl_exec($ch);

curl_close($ch);

print_r($response);
?>

Ruby

require 'net/http'
require 'json'

params = {
  url: "https://www.example.com"
}

uri = URI('https://api.scraperapi.com')
uri.query = URI.encode_www_form(params)

req = Net::HTTP::Get.new(uri)
# Replace the value for api_key with your actual API Key.
req['x-sapi-api_key'] = 'API_KEY'
req['x-sapi-render'] = 'true'

http = Net::HTTP.new(uri.hostname, uri.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_PEER

website_content = http.request(req)
puts website_content.body

Java

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

public class Main {
    public static void main(String[] args) throws Exception {
        // Replace the value for api_key with your actual API Key.
        String apiKey = "API_KEY";
        String targetUrl = "https://www.example.com/";

        String scraperApiUrl = "https://api.scraperapi.com?url=" + targetUrl;

        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request = HttpRequest.newBuilder()
                .uri(URI.create(scraperApiUrl))
                .GET()
                .header("x-sapi-api_key", apiKey)
                .header("x-sapi-render", "true")
                .build();

        HttpResponse<String> response =
                client.send(request, HttpResponse.BodyHandlers.ofString());

        System.out.println(response.body());
    }
}

• PROXY MODE

cURL

curl --request GET \
  --url 'https://www.example.com' \
  --proxy 'http://scraperapi:[e-mail]:8001' \
  --header 'x-sapi-render: true' \
  -k

Python

import requests

url = "https://www.example.com"
# Replace the value for api_key with your actual API Key.
proxy = "http://scraperapi:[e-mail]:8001"

proxies = {
    "http": proxy,
    "https": proxy,
}

headers = {
    "x-sapi-render": "true"
}

response = requests.get(url, headers=headers, proxies=proxies, verify=False)
print(response.text)

NodeJS

import axios from 'axios';
import https from 'https';

const url = 'https://www.example.com';
const agent = new https.Agent({ rejectUnauthorized: false });

axios.get(url, {
  headers: {
    'x-sapi-render': 'true'
  },
  proxy: {
    host: 'proxy-server.scraperapi.com',
    port: 8001,
    auth: {
      username: 'scraperapi',
      // Replace the value for password with your actual API Key.
      password: 'API_KEY'
    },
    protocol: 'http'
  },
  httpsAgent: agent
})
.then(response => {
  console.log(response.data);
})
.catch(error => {
  console.error(error);
});

PHP

<?php

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://www.example.com");
curl_setopt($ch, CURLOPT_PROXY,
    // Replace the value for api_key with your actual API Key.
    "http://scraperapi:[e-mail]:8001"
);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_HTTPHEADER, [
    "x-sapi-render: true"
]);

$response = curl_exec($ch);

if(curl_errno($ch)) {
    echo 'Curl error: ' . curl_error($ch);
} else {
    var_dump($response);
}

curl_close($ch);

Ruby

require 'httparty'

HTTParty::Basement.default_options.update(verify: false)

response = HTTParty.get('https://www.example.com', {
  http_proxyaddr: "proxy-server.scraperapi.com",
  http_proxyport: 8001,
  http_proxyuser: "scraperapi",
  # Replace the value for http_proxypass with your actual API Key.
  http_proxypass: "API_KEY",
  headers: {
    "x-sapi-render" => "true"
  }
})

results = response.body
puts results

Java

import java.io.*;
import java.net.*;
import javax.net.ssl.*;
import java.security.cert.X509Certificate;

public class Main {
    public static void main(String[] args) {
        try {
            SSLContext sc = SSLContext.getInstance("TLS");
            sc.init(null, new TrustManager[]{
                new X509TrustManager() {
                    public X509Certificate[] getAcceptedIssuers() { return null; }
                    public void checkClientTrusted(X509Certificate[] certs, String authType) {}
                    public void checkServerTrusted(X509Certificate[] certs, String authType) {}
                }
            }, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
            HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);

            String apiKey = "API_KEY"; // Replace the value for api_key with your actual API Key.
            Authenticator.setDefault(new Authenticator() {
                protected PasswordAuthentication getPasswordAuthentication() {
                    return new PasswordAuthentication(
                        "scraperapi", apiKey.toCharArray()
                    );
                }
            });

            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
            System.setProperty("https.proxyHost", "proxy-server.scraperapi.com");
            System.setProperty("https.proxyPort", "8001");

            HttpsURLConnection conn = (HttpsURLConnection) new URL("https://www.example.com/").openConnection();
            conn.setRequestMethod("GET");

            conn.setRequestProperty("x-sapi-render", "true");

            BufferedReader in;
            if (conn.getResponseCode() >= 400) {
                in = new BufferedReader(new InputStreamReader(conn.getErrorStream()));
            } else {
                in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
            }

            StringBuilder resp = new StringBuilder();
            String line;
            while ((line = in.readLine()) != null) {
                resp.append(line).append("\n");
            }
            in.close();

            System.out.println(resp);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Credentials must still be passed to the proxy API using the standard method, not as headers.