Parameters as Headers

Learn how to scrape websites using header-based parameters with ScraperAPI in Java. Pass api_key, render, and instruction_set via headers to simplify requests.

Along with the “traditional” means of passing parameters, we also support passing parameters as headers. Passing parameters such as api_key, render, ultra_premium and instruction_set is very straightforward.

• API REQUEST

Instead of including the parameters in the URL

import java.net.*;
import java.io.*;

public class Scrape {
  public static void main(String[] args) {

    try {
      String url = "https://api.scraperapi.com/?api_key=<YOUR_API_KEY>&url=https%3A%2F%2Fhttpbin.org%2Fip&render=true";
      URL urlForGetRequest = new URL(url);
      String readLine = null;
      HttpURLConnection conection = (HttpURLConnection) urlForGetRequest.openConnection();
      conection.setRequestMethod("GET");
      int responseCode = conection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        BufferedReader in = new BufferedReader(new InputStreamReader(conection.getInputStream()));
        StringBuffer response = new StringBuffer();
        while ((readLine = in.readLine()) != null) {
          response.append(readLine);
        }
        in.close();
        System.out.println(response.toString());
      } else {
        throw new Exception("Error in API Call");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }
}

you can just pass them as headers

import java.net.*;
import java.io.*;

public class Scrape {
  public static void main(String[] args) {
    try {
      String url = "https://api.scraperapi.com?url=https://httpbin.org/ip";

      URL urlForGetRequest = new URL(url);
      String readLine = null;
      HttpURLConnection connection = (HttpURLConnection) urlForGetRequest.openConnection();
      connection.setRequestMethod("GET");
      connection.setRequestProperty("x-sapi-render", "true");
      connection.setRequestProperty("x-sapi-api_key", "<YOUR_API_KEY>");
      int responseCode = connection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        StringBuffer response = new StringBuffer();
        while ((readLine = in.readLine()) != null) {
          response.append(readLine);
        }
        in.close();
        System.out.println(response.toString());
      } else {
        throw new Exception("Error in API Call");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }
}

Please note that the 'x-sapi-' prefix is used on each header to avoid collisions with headers used by target sites. We support all standard parameters available with the API. The instruction_set parameter is specifically supported only through headers.

• PROXY MODE

import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.HttpURLConnection;
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Properties;

public class Scrape {
    public static void main(String[] args) {
        try {
            String apiKey = "<YOUR_API_KEY>";
            String proxyHost = "proxy-server.scraperapi.com";
            int proxyPort = 8001;
            String proxyUser = "scraperapi";
            String proxyPassword = apiKey;

            // Set proxy properties
            Properties systemProperties = System.getProperties();
            systemProperties.setProperty("http.proxyHost", proxyHost);
            systemProperties.setProperty("http.proxyPort", String.valueOf(proxyPort));

            // Set authenticator for proxy authentication
            Authenticator.setDefault(new Authenticator() {
                protected PasswordAuthentication getPasswordAuthentication() {
                    if (getRequestorType().equals(RequestorType.PROXY)) {
                        return new PasswordAuthentication(proxyUser, proxyPassword.toCharArray());
                    }
                    return null;
                }
            });

            String apiUrl = "http://httpbin.org/ip";
            URL server = new URL(apiUrl);
            HttpURLConnection connection = (HttpURLConnection) server.openConnection();
            connection.setRequestProperty("x-sapi-render", "true");

            connection.connect();

            int responseCode = connection.getResponseCode();

            if (responseCode == HttpURLConnection.HTTP_OK) {
                BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                StringBuffer response = new StringBuffer();
                String inputLine;

                while ((inputLine = in.readLine()) != null) {
                    response.append(inputLine);
                }
                in.close();

                System.out.println(response.toString());
            } else {
                throw new Exception("Error in API Call, Response Code: " + responseCode);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}

Note that credentials must still be passed to the proxy in the manner shown above, not as headers.