Job Handling

Submitting a Job

Using the Async API is easy. Submit a scraping job and receive a status URL, where you can monitor the job and collect the results once it's finished.

1

Base URL

https://async.scraperapi.com/jobs

2

Required query parameters

• api_key - your API Key

• url - target URL

3

Sample Request

cURL

curl --request POST \
  --url "https://async.scraperapi.com/jobs" \
  --header "Content-Type: application/json" \
  --data '{
    "apiKey": "API_KEY",
    "url": "https://example.com"
  }'

Python

import requests

r = requests.post(url = 'https://async.scraperapi.com/jobs',
json={ 'apiKey': 'API_KEY', 'url': 'https://example.com' })

print(r.text)

NodeJS

import axios from 'axios';

(async () => {
  const { data } = await axios({
    method: 'POST',
    url: 'https://async.scraperapi.com/jobs',
    headers: { 'Content-Type': 'application/json' },
    data: {
      apiKey: 'API_KEY',
      url: 'https://example.com'
    }
  });

  console.log(data);
})();

PHP

<?php
$payload = json_encode([
    "apiKey" => "API_KEY",
    "url"    => "https://example.com"
]);

$ch = curl_init("https://async.scraperapi.com/jobs");

curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Content-Type: application/json"]);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

print_r($response);

Ruby

require 'net/http'
require 'json'

uri = URI('https://async.scraperapi.com/jobs')

website_content = Net::HTTP.post(uri, { "apiKey" => "API_KEY", "url" => "https://example.com"}.to_json, "Content-Type" => "application/json")

print(website_content.body)

Java

import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) {
        try {
            URL url = new URL("https://async.scraperapi.com/jobs");
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("POST");
            conn.setRequestProperty("Content-Type", "application/json");
            conn.setDoOutput(true);

            String payload = "{\"apiKey\": \"API_KEY\", \"url\": \"https://example.com\"}";
            try (OutputStream os = conn.getOutputStream()) {
                os.write(payload.getBytes(StandardCharsets.UTF_8));
            }

            try (BufferedReader in = new BufferedReader(new InputStreamReader(
                    conn.getResponseCode() == 200 ? conn.getInputStream() : conn.getErrorStream()
            ))) {
                StringBuilder response = new StringBuilder();
                String line;
                while ((line = in.readLine()) != null) response.append(line);
                System.out.println(response);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Response:

}
"id":"0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"status":"running",
"statusUrl":"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"url":"https://example.com"
}

Sending a POST request to the Async API is done by using "method": "POST" inside the payload. Here is an example:

cURL

curl --request POST \
  --url "https://async.scraperapi.com/jobs" \
  --header "Content-Type: application/json" \
  --data '{
    "apiKey": "API_KEY",
    "url": "https://postman-echo.com/post",
    "method": "POST",
    "headers": {
      "content-type": "application/x-www-form-urlencoded"
    },
    "body": "foo=bar"
  }'

Python

import requests

body = {'apiKey': 'API_KEY', 'url': 'https://postman-echo.com/post', 'method': 'POST', 'headers': {"content-type": "application/x-www-form-urlencoded"},  'body': 'foo=bar'}

r = requests.post('https://async.scraperapi.com/jobs', headers={"Content-Type": "application/json"},  json=body)
print(r.text)

NodeJS

const url = 'https://async.scraperapi.com/jobs';

const body = {
  apiKey: 'API_KEY',
  url: 'https://postman-echo.com/post',
  method: 'POST',
  headers: { "content-type": "application/x-www-form-urlencoded" },
  body: "foo=bar"
};

(async () => {
  try {
    const res = await fetch(url, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(body)
    });
    const data = await res.json();
    console.log(data);
  } catch (err) {
    console.error(err);
  }
})();

PHP

<?php
$url = "https://async.scraperapi.com/jobs";

$postData = json_encode([
    "url" => "https://postman-echo.com/post",
    "apiKey" => "API_KEY",
    "method" => "POST",
    "headers" => [ "Content-Type" => "application/x-www-form-urlencoded" ],
    "body" => "foo=bar"
]);

$headers = [ "Content-Type: application/json" ];

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$response = curl_exec($ch);
curl_close($ch);

print_r($response);

Ruby

require 'net/http'
require 'json'


uri = URI('https://async.scraperapi.com/jobs')
body = {
 "apiKey" => "API_KEY",
 "url" => "https://postman-echo.com/post",
 "method" =>  "POST",
 "headers" => {"content-type" => "application/x-www-form-urlencoded"} ,
 "body" => "foo=bar"
}

response = Net::HTTP.post(uri, body.to_json, "Content-Type" => "application/json")
print(response.body)

Java

import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) {
        try {
            HttpURLConnection conn = (HttpURLConnection) new URL("https://async.scraperapi.com/jobs").openConnection();
            conn.setRequestMethod("POST");
            conn.setRequestProperty("Content-Type", "application/json");
            conn.setDoOutput(true);
            conn.getOutputStream().write("{\"apiKey\":\"API_KEY\",\"url\":\"https://postman-echo.com/post\",\"method\":\"POST\",\"headers\":{\"content-type\":\"text/plain\"},\"body\":\"foo=bar\"}".getBytes(StandardCharsets.UTF_8));

            BufferedReader in = new BufferedReader(new InputStreamReader(
                    conn.getResponseCode() == 200 ? conn.getInputStream() : conn.getErrorStream()
            ));
            StringBuilder res = new StringBuilder(); String line;
            while ((line = in.readLine()) != null) res.append(line);
            System.out.println(res);
        } catch (Exception e) { e.printStackTrace(); }
    }
}

Should you wish, you can include meta object in your request to store custom data (your own request ID for example), which will be returned in the response as well.

Job Status

The statusUrl is a unique job URL, used to retrieve the status and results of the scraping job. Invoking that endpoint provides you with the job status (while it is still running)

cURL

curl "https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953"

Python

import requests

r = requests.get(url = 'https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953')
print(r.text)

NodeJS

const url = 'https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953';

fetch(url)
  .then(res => res.text())
  .then(text => console.log(text))
  .catch(err => console.error(err));

PHP

<?php
$ch = curl_init(); curl_setopt( $ch, CURLOPT_URL,
"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953"
); 
curl_setopt( $ch, CURLOPT_RETURNTRANSFER,TRUE); 

$response = curl_exec( $ch ); 
curl_close( $ch ); 

print_r( $response );

Ruby

require 'net/http'
require 'json'

uri = URI('https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953')

website_content = Net::HTTP.get(uri)
print(website_content)

Java

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class Main {
    public static void main(String[] args) {
        try {
            URL asyncApiUrl = new URL("https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953");
            HttpURLConnection connection = (HttpURLConnection) asyncApiUrl.openConnection();
            connection.setRequestMethod("GET");

            int responseCode = connection.getResponseCode();

            try (BufferedReader in = new BufferedReader(new InputStreamReader(
                    responseCode == HttpURLConnection.HTTP_OK ? connection.getInputStream() : connection.getErrorStream()
            ))) {
                StringBuilder response = new StringBuilder();
                String line;
                while ((line = in.readLine()) != null) {
                    response.append(line);
                }
                System.out.println(response);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}

Once your job is finished, the response from the statusURL will this time contain the results of your scraping job:

{
"id":"0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"status":"finished",
"statusUrl":"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"url":"https://example.com",
"response":{
"headers":{
"date":"Thu, 14 Apr 2022 11:10:44 GMT",
"content-type":"text/html; charset=utf-8",
"content-length":"1256",
"connection":"close",
"x-powered-by":"Express",
"access-control-allow-origin":"undefined","access-control-allow-headers":"Origin, X-Requested-With, Content-Type, Accept",
"access-control-allow-methods":"HEAD,GET,POST,DELETE,OPTIONS,PUT",
"access-control-allow-credentials":"true",
"x-robots-tag":"none",
"sa-final-url":"https://example.com/",
"sa-statuscode":"200",
"etag":"W/\"4e8-Sjzo7hHgkd15I/TYxuW15B7HwEc\"",
"vary":"Accept-Encoding"
},
"body":"<!doctype html>\n<html>\n<head>\n	<title>Example Domain</title>\n\n	<meta charset=\"utf-8\" />\n	<meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n	<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n	<style type=\"text/css\">\n	body {\n		background-color: #f0f0f2;\n		margin: 0;\n		padding: 0;\n		font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\", \"Open Sans\", \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n		\n	}\n	div {\n		width: 600px;\n		margin: 5em auto;\n		padding: 2em;\n		background-color: #fdfdff;\n		border-radius: 0.5em;\n		box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n	}\n	a:link, a:visited {\n		color: #38488f;\n		text-decoration: none;\n	}\n	@media (max-width: 700px) {\n		div {\n			margin: 0 auto;\n			width: auto;\n		}\n	}\n	</style>	\n</head>\n\n<body>\n<div>\n	<h1>Example Domain</h1>\n	<p>This domain is for use in illustrative examples in documents. You may use this\n	domain in literature without prior coordination or asking for permission.</p>\n	<p><a href=\"https://www.iana.org/domains/example\">More information...</a></p>\n</div>\n</body>\n</html>\n",
"statusCode":200
}
}

Please note that the response for an Async job is stored for up to 72 hours (24hrs guaranteed) or until you retrieve the results, whichever comes first. If you do not get the response in due time, it will be deleted from our side and you will have to send another request for the same job.

Cancelling a Job

Should you wish to cancel a running job, you can do so by sending a DELETE request to the job endpoint using the job ID:

curl -X DELETE https://async.scraperapi.com/jobs/<jobId>