Making POST/PUT Requests

In addition to standard GET requests, ScraperAPI lets you send POST/PUT requests as well. This makes it possible to scrape APIs, submit forms and send data to endpoints that require more than just a URL.

• API REQUEST

cURL

# Replace POST with PUT to send a PUT request instead
curl -d 'foo=bar' \
-X POST \
"http://api.scraperapi.com/?api_key=API_KEY&url=https://postman-echo.com/post"

# For form data
curl -H 'Content-Type: application/x-www-form-urlencoded' \
-F 'foo=bar' \
-X POST \
"http://api.scraperapi.com/?api_key=API_KEY&url=https://postman-echo.com/post"

Python

import requests
payload = {'api_key': 'API_KEY', 'url': 'https://postman-echo.com/post'}

r = requests.post('http://api.scraperapi.com', params=payload, data={'foo': 'bar'})
print(r.text)

NodeJS

import fetch from 'node-fetch';

const url = 'http://api.scraperapi.com/?api_key=API_KEY&url=https://postman-echo.com/post';

const options = {
  method: 'POST',
  body: JSON.stringify({ foo: 'bar' }),
  headers: {
    'Content-Type': 'application/json',
  },
};

fetch(url, options)
  .then(response => response.json())
  .then(data => {
    console.log(data);
  })
  .catch(error => {
    console.error('Error:', error);
  });

PHP

<?php
$url = "http://api.scraperapi.com?api_key=API_KEY&url=https://postman-echo.com/post";

# POST/PUT Request
$postData = ["foo" =>"bar"]; 
$postData = json_encode($postData);
$headers = [
    "Content-Type: application/json"
];  
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_POST,1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);

curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
$response = curl_exec($ch);
curl_close($ch);
print_r($response);

Ruby

require 'net/http'
require 'json'

## Replace POST with PUT to send a PUT request instead
params = {
:api_key => "API_KEY",
:url => "https://postman-echo.com/post"
}

uri = URI('http://api.scraperapi.com/')
uri.query = URI.encode_www_form(params)

website_content = Net::HTTP.post(uri, { "foo" => "bar"}.to_json, "Content-Type" => "application/json")
print(website_content.body)

Java

import java.io.OutputStream;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class Main {
    public static void main(String[] args) {
        try {
            String apiKey = "API_KEY";
            String urlStr = "https://api.scraperapi.com/?api_key=" + apiKey + "&url=https://postman-echo.com/post";
            URL url = new URL(urlStr);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("POST");
            conn.setDoOutput(true);
            conn.setRequestProperty("Content-Type", "application/json");

            String jsonInput = "{\"foo\":\"bar\"}";
            try (OutputStream os = conn.getOutputStream()) {
                os.write(jsonInput.getBytes());
                os.flush();
            }

            int responseCode = conn.getResponseCode();
            if (responseCode == HttpURLConnection.HTTP_OK) {
                BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
                StringBuilder response = new StringBuilder();
                String line;
                while ((line = in.readLine()) != null) {
                    response.append(line);
                }
                in.close();
                System.out.println(response.toString());
            } else {
                System.out.println("Error in API Call: " + responseCode);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}

• PROXY MODE

cURL

# Replace POST with PUT to send a PUT request instead
curl -d 'foo=bar' \
-X POST \
-x "http://scraperapi:[e-mail]:8001" -k "https://postman-echo.com/post"

# For form data
curl -H 'Content-Type: application/x-www-form-urlencoded' \
-F 'foo=bar' \
-X POST \
-x "http://scraperapi:[e-mail]:8001" -k "https://postman-echo.com/post"

Python

import requests
proxies = {
  "http": "http://scraperapi:[e-mail]:8001"
}

r = requests.post('https://postman-echo.com/post', proxies=proxies, data={'foo': 'bar'}, verify=False)
print(r.text)

NodeJS

import axios from 'axios';

axios.post(
  'https://postman-echo.com/post',
  { foo: 'bar' },
  {
    headers: {
      'Content-Type': 'application/json',
    },
    proxy: {
      host: 'proxy-server.scraperapi.com',
      port: 8001,
      auth: {
        username: 'scraperapi',
        password: 'API_KEY',
      },
      protocol: 'http',
    },
  }
)
  .then(response => {
    console.log(response.data);
  })
  .catch(error => {
    console.error('Error:', error.message);
  });

PHP

<?php
$postData = ["foo"=>"bar"];
$postData = json_encode($postData);
$headers = [
  "Content-Type: application/json"
];

# POST/PUT Request
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://postman-echo.com/post");
curl_setopt($ch, CURLOPT_PROXY, "http://scraperapi:[e-mail]:8001");
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData); 
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

$response = curl_exec($ch);
curl_close($ch);

echo $response;

Ruby

require 'httparty'
HTTParty::Basement.default_options.update(verify: false)
query = {'foo': 'bar'}

## POST/PUT Request
headers = {'Accept':'application/json', 'Content-Type':'application/json'}

response = HTTParty.post('https://postman-echo.com/post', {
  http_proxyaddr: "proxy-server.scraperapi.com",
  http_proxyport: "8001",
  http_proxyuser: "scraperapi",
  http_proxypass: "API_KEY",
  body: query.to_json,
  headers: headers
}, )
results = response.body
puts results

Java

import java.io.*;
import java.net.*;
import javax.net.ssl.*;
import javax.net.ssl.X509TrustManager;

public class Main {
    public static void main(String[] args) {
        try {
            SSLContext sc = SSLContext.getInstance("TLS");
            sc.init(null, new TrustManager[]{new X509TrustManager() {
                public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; }
                public void checkClientTrusted(java.security.cert.X509Certificate[] c, String a) {}
                public void checkServerTrusted(java.security.cert.X509Certificate[] c, String a) {}
            }}, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
            HttpsURLConnection.setDefaultHostnameVerifier((h, s) -> true);

            String apiKey = "API_KEY"; // Replace with your actual API Key
            Authenticator.setDefault(new Authenticator() {
                protected PasswordAuthentication getPasswordAuthentication() {
                    return new PasswordAuthentication("scraperapi", apiKey.toCharArray());
                }
            });

            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
            System.setProperty("https.proxyHost", "proxy-server.scraperapi.com");
            System.setProperty("https.proxyPort", "8001");

            URL url = new URL("https://postman-echo.com/post");
            HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
            conn.setRequestMethod("POST");
            conn.setDoOutput(true);
            conn.setRequestProperty("Content-Type", "application/json");

            String jsonInput = "{\"foo\":\"bar\"}";
            try (OutputStream os = conn.getOutputStream()) {
                os.write(jsonInput.getBytes());
            }

            try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
                StringBuilder resp = new StringBuilder();
                for (String line; (line = in.readLine()) != null; ) resp.append(line).append("\n");
                System.out.println(resp.toString());
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}