Rendering Javascript

If you are crawling a page that requires you to render the Javascript on the page to scrape the data you need, then we can fetch these pages using a headless browser.

To render Javascript, simply set render=true and we will use a headless Google Chrome instance to fetch the page. This feature is available on all plans.

Pass the JavaScript rendering parameter within the URL:

• API REQUEST

import java.net.*;
import java.io.*;

public class Scrape {
  public static void main(String[] args) {

    try {
      String url = "https://api.scraperapi.com/?api_key=<YOUR_API_KEY>&url=https%3A%2F%2Fhttpbin.org%2Fip&render=true";
      URL urlForGetRequest = new URL(url);
      String readLine = null;
      HttpURLConnection conection = (HttpURLConnection) urlForGetRequest.openConnection();
      conection.setRequestMethod("GET");
      int responseCode = conection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        BufferedReader in = new BufferedReader(new InputStreamReader(conection.getInputStream()));
        StringBuffer response = new StringBuffer();
        while ((readLine = in.readLine()) != null) {
          response.append(readLine);
        }
        in.close();
        System.out.println(response.toString());
      } else {
        throw new Exception("Error in API Call");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }
}

• PROXY MODE

import java.net.*;
import java.io.*;
import java.util.*;

public class Scrape {
  public static void main(String[] args) {
    try {
      String apiKey = "YOUR_API_KEY";
      String proxy = "proxy-server.scraperapi.com";
      URL server = new URL("https://httpbin.org/ip");
      String proxyUsername = "scraperapi";
      Authenticator.setDefault(new Authenticator() {
        protected PasswordAuthentication getPasswordAuthentication() {
          return new PasswordAuthentication(proxyUsername, apiKey.toCharArray());
        }
      });
      Properties systemProperties = System.getProperties();
      systemProperties.setProperty("http.proxyHost", proxy);
      systemProperties.setProperty("http.proxyPort", "8001");
      systemProperties.setProperty("https.proxyHost", proxy);
      systemProperties.setProperty("https.proxyPort", "{{PROXY_PORT}}");
      System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

      HttpURLConnection httpURLConnection = (HttpURLConnection) server.openConnection();
      httpURLConnection.connect();
      String readLine = null;
      int responseCode = httpURLConnection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        BufferedReader in = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream()));
        StringBuffer response = new StringBuffer();
        while ((readLine = in.readLine()) != null) {
          response.append(readLine);
        }
        in.close();
        System.out.println(response.toString());
      } else {
        throw new Exception("Error in API Call");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }
}

• SDK Method

// remember to install the library: https://search.maven.org/artifact/com.scraperapi/sdk/1.0

package com.example;

import com.scraperapi.ScraperApiClient;

public class Scrape {

    public static void main(String[] args) {
      ScraperApiClient client = new ScraperApiClient("YOUR_API_KEY");
      String result = client.get("http://httpbin.org/ip")
        .render(true)
        .result();

      System.out.println(result);
    }
}

Pass the parameter in the headers:

• API REQUEST

import java.net.*;
import java.io.*;

public class Scrape {
  public static void main(String[] args) {
    try {
      String url = "https://api.scraperapi.com?url=https://httpbin.org/ip";

      URL urlForGetRequest = new URL(url);
      String readLine = null;
      HttpURLConnection connection = (HttpURLConnection) urlForGetRequest.openConnection();
      connection.setRequestMethod("GET");
      connection.setRequestProperty("x-sapi-render", "true");
      connection.setRequestProperty("x-sapi-api_key", "<YOUR_API_KEY>");
      int responseCode = connection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        StringBuffer response = new StringBuffer();
        while ((readLine = in.readLine()) != null) {
          response.append(readLine);
        }
        in.close();
        System.out.println(response.toString());
      } else {
        throw new Exception("Error in API Call");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }
}

• PROXY MODE

import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.HttpURLConnection;
import java.net.URL;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Properties;

public class Scrape {
    public static void main(String[] args) {
        try {
            String apiKey = "<YOUR_API_KEY>";
            String proxyHost = "proxy-server.scraperapi.com";
            int proxyPort = 8001;
            String proxyUser = "scraperapi";
            String proxyPassword = apiKey;

            // Set proxy properties
            Properties systemProperties = System.getProperties();
            systemProperties.setProperty("http.proxyHost", proxyHost);
            systemProperties.setProperty("http.proxyPort", String.valueOf(proxyPort));

            // Set authenticator for proxy authentication
            Authenticator.setDefault(new Authenticator() {
                protected PasswordAuthentication getPasswordAuthentication() {
                    if (getRequestorType().equals(RequestorType.PROXY)) {
                        return new PasswordAuthentication(proxyUser, proxyPassword.toCharArray());
                    }
                    return null;
                }
            });

            String apiUrl = "http://httpbin.org/ip";
            URL server = new URL(apiUrl);
            HttpURLConnection connection = (HttpURLConnection) server.openConnection();
            connection.setRequestProperty("x-sapi-render", "true");

            connection.connect();

            int responseCode = connection.getResponseCode();

            if (responseCode == HttpURLConnection.HTTP_OK) {
                BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                StringBuffer response = new StringBuffer();
                String inputLine;

                while ((inputLine = in.readLine()) != null) {
                    response.append(inputLine);
                }
                in.close();

                System.out.println(response.toString());
            } else {
                throw new Exception("Error in API Call, Response Code: " + responseCode);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}

• SKD Method

package com.example;

import com.scraperapi.ScraperApiClient;

public class Scrape {

    public static void main(String[] args) {
        ScraperApiClient client = new ScraperApiClient("<YOUR_API_KEY>");
        client.setHeader("x-scraperapi-render", "true");
        String result = client.get("http://httpbin.org/ip").result();
        System.out.println(result);
    }
}