Job Handling

Submitting a Job

Using the Async API is easy. Submit a scraping job and receive a status URL, where you can monitor the job and collect the results once it's finished.

Base URL

https://async.scraperapi.com/jobs

Required query parameters

api_key - your API Key
url - target URL

Sample Request

curl --request POST \
  --url "https://async.scraperapi.com/jobs" \
  --header "Content-Type: application/json" \
  --data '{
    "apiKey": "API_KEY",
    "url": "https://example.com"
  }'

import axios from 'axios';

(async () => {
  const { data } = await axios({
    method: 'POST',
    url: 'https://async.scraperapi.com/jobs',
    headers: { 'Content-Type': 'application/json' },
    data: {
      apiKey: 'API_KEY',
      url: 'https://example.com'
    }
  });

  console.log(data);
})();

<?php
$payload = json_encode([
    "apiKey" => "API_KEY",
    "url"    => "https://example.com"
]);

$ch = curl_init("https://async.scraperapi.com/jobs");

curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Content-Type: application/json"]);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

print_r($response);

require 'net/http'
require 'json'

uri = URI('https://async.scraperapi.com/jobs')

website_content = Net::HTTP.post(uri, { "apiKey" => "API_KEY", "url" => "https://example.com"}.to_json, "Content-Type" => "application/json")

print(website_content.body)

import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) {
        try {
            URL url = new URL("https://async.scraperapi.com/jobs");
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("POST");
            conn.setRequestProperty("Content-Type", "application/json");
            conn.setDoOutput(true);

            String payload = "{\"apiKey\": \"API_KEY\", \"url\": \"https://example.com\"}";
            try (OutputStream os = conn.getOutputStream()) {
                os.write(payload.getBytes(StandardCharsets.UTF_8));
            }

            try (BufferedReader in = new BufferedReader(new InputStreamReader(
                    conn.getResponseCode() == 200 ? conn.getInputStream() : conn.getErrorStream()
            ))) {
                StringBuilder response = new StringBuilder();
                String line;
                while ((line = in.readLine()) != null) response.append(line);
                System.out.println(response);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Response:

}
"id":"0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"status":"running",
"statusUrl":"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"url":"https://example.com"
}

Sending a POST request to the Async API is done by using "method": "POST" inside the payload. Here is an example:

curl --request POST \
  --url "https://async.scraperapi.com/jobs" \
  --header "Content-Type: application/json" \
  --data '{
    "apiKey": "API_KEY",
    "url": "https://postman-echo.com/post",
    "method": "POST",
    "headers": {
      "content-type": "application/x-www-form-urlencoded"
    },
    "body": "foo=bar"
  }'

import requests

body = {'apiKey': 'API_KEY', 'url': 'https://postman-echo.com/post', 'method': 'POST', 'headers': {"content-type": "application/x-www-form-urlencoded"},  'body': 'foo=bar'}

r = requests.post('https://async.scraperapi.com/jobs', headers={"Content-Type": "application/json"},  json=body)
print(r.text)

const url = 'https://async.scraperapi.com/jobs';

const body = {
  apiKey: 'API_KEY',
  url: 'https://postman-echo.com/post',
  method: 'POST',
  headers: { "content-type": "application/x-www-form-urlencoded" },
  body: "foo=bar"
};

(async () => {
  try {
    const res = await fetch(url, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(body)
    });
    const data = await res.json();
    console.log(data);
  } catch (err) {
    console.error(err);
  }
})();

<?php
$url = "https://async.scraperapi.com/jobs";

$postData = json_encode([
    "url" => "https://postman-echo.com/post",
    "apiKey" => "API_KEY",
    "method" => "POST",
    "headers" => [ "Content-Type" => "application/x-www-form-urlencoded" ],
    "body" => "foo=bar"
]);

$headers = [ "Content-Type: application/json" ];

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);

$response = curl_exec($ch);
curl_close($ch);

print_r($response);

require 'net/http'
require 'json'


uri = URI('https://async.scraperapi.com/jobs')
body = {
 "apiKey" => "API_KEY",
 "url" => "https://postman-echo.com/post",
 "method" =>  "POST",
 "headers" => {"content-type" => "application/x-www-form-urlencoded"} ,
 "body" => "foo=bar"
}

response = Net::HTTP.post(uri, body.to_json, "Content-Type" => "application/json")
print(response.body)

import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;

public class Main {
    public static void main(String[] args) {
        try {
            HttpURLConnection conn = (HttpURLConnection) new URL("https://async.scraperapi.com/jobs").openConnection();
            conn.setRequestMethod("POST");
            conn.setRequestProperty("Content-Type", "application/json");
            conn.setDoOutput(true);
            conn.getOutputStream().write("{\"apiKey\":\"API_KEY\",\"url\":\"https://postman-echo.com/post\",\"method\":\"POST\",\"headers\":{\"content-type\":\"text/plain\"},\"body\":\"foo=bar\"}".getBytes(StandardCharsets.UTF_8));

            BufferedReader in = new BufferedReader(new InputStreamReader(
                    conn.getResponseCode() == 200 ? conn.getInputStream() : conn.getErrorStream()
            ));
            StringBuilder res = new StringBuilder(); String line;
            while ((line = in.readLine()) != null) res.append(line);
            System.out.println(res);
        } catch (Exception e) { e.printStackTrace(); }
    }
}

Should you wish, you can include meta object in your request to store custom data (your own request ID for example), which will be returned in the response as well.

Job Status

The statusUrl is a unique job URL, used to retrieve the status and results of the scraping job. Invoking that endpoint provides you with the job status (while it is still running)

curl "https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953"

const url = 'https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953';

fetch(url)
  .then(res => res.text())
  .then(text => console.log(text))
  .catch(err => console.error(err));

<?php
$ch = curl_init(); curl_setopt( $ch, CURLOPT_URL,
"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953"
); 
curl_setopt( $ch, CURLOPT_RETURNTRANSFER,TRUE); 

$response = curl_exec( $ch ); 
curl_close( $ch ); 

print_r( $response );

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class Main {
    public static void main(String[] args) {
        try {
            URL asyncApiUrl = new URL("https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953");
            HttpURLConnection connection = (HttpURLConnection) asyncApiUrl.openConnection();
            connection.setRequestMethod("GET");

            int responseCode = connection.getResponseCode();

            try (BufferedReader in = new BufferedReader(new InputStreamReader(
                    responseCode == HttpURLConnection.HTTP_OK ? connection.getInputStream() : connection.getErrorStream()
            ))) {
                StringBuilder response = new StringBuilder();
                String line;
                while ((line = in.readLine()) != null) {
                    response.append(line);
                }
                System.out.println(response);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}

Once your job is finished, the response from the statusURL will this time contain the results of your scraping job:

{
"id":"0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"status":"finished",
"statusUrl":"https://async.scraperapi.com/jobs/0962a8e0-5f1a-4e14-bf8c-5efcc18f0953",
"url":"https://example.com",
"response":{
"headers":{
"date":"Thu, 14 Apr 2022 11:10:44 GMT",
"content-type":"text/html; charset=utf-8",
"content-length":"1256",
"connection":"close",
"x-powered-by":"Express",
"access-control-allow-origin":"undefined","access-control-allow-headers":"Origin, X-Requested-With, Content-Type, Accept",
"access-control-allow-methods":"HEAD,GET,POST,DELETE,OPTIONS,PUT",
"access-control-allow-credentials":"true",
"x-robots-tag":"none",
"sa-final-url":"https://example.com/",
"sa-statuscode":"200",
"etag":"W/\"4e8-Sjzo7hHgkd15I/TYxuW15B7HwEc\"",
"vary":"Accept-Encoding"
},
"body":"<!doctype html>\n<html>\n<head>\n	<title>Example Domain</title>\n\n	<meta charset=\"utf-8\" />\n	<meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n	<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n	<style type=\"text/css\">\n	body {\n		background-color: #f0f0f2;\n		margin: 0;\n		padding: 0;\n		font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\", \"Open Sans\", \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n		\n	}\n	div {\n		width: 600px;\n		margin: 5em auto;\n		padding: 2em;\n		background-color: #fdfdff;\n		border-radius: 0.5em;\n		box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n	}\n	a:link, a:visited {\n		color: #38488f;\n		text-decoration: none;\n	}\n	@media (max-width: 700px) {\n		div {\n			margin: 0 auto;\n			width: auto;\n		}\n	}\n	</style>	\n</head>\n\n<body>\n<div>\n	<h1>Example Domain</h1>\n	<p>This domain is for use in illustrative examples in documents. You may use this\n	domain in literature without prior coordination or asking for permission.</p>\n	<p><a href=\"https://www.iana.org/domains/example\">More information...</a></p>\n</div>\n</body>\n</html>\n",
"statusCode":200
}
}

Please note that the response for an Async job is stored for up to 72 hours (24hrs guaranteed) or until you retrieve the results, whichever comes first. If you do not get the response in due time, it will be deleted from our side and you will have to send another request for the same job.

Cancelling a Job

Should you wish to cancel a running job, you can do so by sending a DELETE request to the job endpoint using the job ID:

curl -X DELETE https://async.scraperapi.com/jobs/<jobId>

PreviousOverview NextBatch Requests

Last updated 2 days ago

hashtagSubmitting a Job

hashtagBase URL

hashtagRequired query parameters

hashtagSample Request

hashtagJob Status

hashtagCancelling a Job

Submitting a Job

Base URL

Required query parameters

Sample Request

Job Status

Cancelling a Job