Get crawl errors and skipped URLs
curl --request GET \
--url https://api.prefetch.io/crawl/{id}/errors \
--header 'X-API-Key: <api-key>'import requests
url = "https://api.prefetch.io/crawl/{id}/errors"
headers = {"X-API-Key": "<api-key>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {'X-API-Key': '<api-key>'}};
fetch('https://api.prefetch.io/crawl/{id}/errors', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.prefetch.io/crawl/{id}/errors",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.prefetch.io/crawl/{id}/errors"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("X-API-Key", "<api-key>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.prefetch.io/crawl/{id}/errors")
.header("X-API-Key", "<api-key>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.prefetch.io/crawl/{id}/errors")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["X-API-Key"] = '<api-key>'
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"id": "crw_9f2c1a7b4e0d4c3a8b1e5f7d2c9a0b3e",
"status": "completed",
"failed": [
{
"url": "https://docs.stripe.com/legacy/v1",
"depth": 2,
"status_code": null,
"error": "Failed to load page (HTTP 503)",
"error_code": "TARGET_SERVER_ERROR"
}
],
"failed_total": 1,
"skipped": {
"out-of-scope": {
"count": 41,
"samples": [
"https://docs.stripe.com/blog/2024",
"https://docs.stripe.com/jobs"
]
},
"robots": {
"count": 3,
"samples": [
"https://docs.stripe.com/search"
]
}
},
"next": null
},
"meta": {
"requestId": "4c1e2a9b-7d3f-4b8e-9a6c-1f2e3d4c5b6a",
"durationMs": 41
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}Web extraction
GET /crawl/{id}/errors
Why a crawl returned fewer pages than expected: failed pages with error codes, and skipped URLs by reason. Free.
GET
/
crawl
/
{id}
/
errors
Get crawl errors and skipped URLs
curl --request GET \
--url https://api.prefetch.io/crawl/{id}/errors \
--header 'X-API-Key: <api-key>'import requests
url = "https://api.prefetch.io/crawl/{id}/errors"
headers = {"X-API-Key": "<api-key>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {'X-API-Key': '<api-key>'}};
fetch('https://api.prefetch.io/crawl/{id}/errors', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.prefetch.io/crawl/{id}/errors",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"X-API-Key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.prefetch.io/crawl/{id}/errors"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("X-API-Key", "<api-key>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.prefetch.io/crawl/{id}/errors")
.header("X-API-Key", "<api-key>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.prefetch.io/crawl/{id}/errors")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["X-API-Key"] = '<api-key>'
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"id": "crw_9f2c1a7b4e0d4c3a8b1e5f7d2c9a0b3e",
"status": "completed",
"failed": [
{
"url": "https://docs.stripe.com/legacy/v1",
"depth": 2,
"status_code": null,
"error": "Failed to load page (HTTP 503)",
"error_code": "TARGET_SERVER_ERROR"
}
],
"failed_total": 1,
"skipped": {
"out-of-scope": {
"count": 41,
"samples": [
"https://docs.stripe.com/blog/2024",
"https://docs.stripe.com/jobs"
]
},
"robots": {
"count": 3,
"samples": [
"https://docs.stripe.com/search"
]
}
},
"next": null
},
"meta": {
"requestId": "4c1e2a9b-7d3f-4b8e-9a6c-1f2e3d4c5b6a",
"durationMs": 41
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}{
"error": "Credit limit exceeded",
"error_code": "CREDITS_EXHAUSTED",
"retryable": false,
"success": false,
"retryAfterSeconds": 123,
"limit": 123,
"meta": {
"requestId": "a3f2c1d4-7b6e-4f2a-9c1d-8e3f2a1b4c5d",
"durationMs": 1842
}
}Overview
Explains a crawl that came back smaller than you expected. It returns two things:failed— pages that were fetched and failed, oldest first, each with anerror_code. Paged withskipandlimit; follownextuntil it isnull.skipped— every URL the crawl found but did not fetch, counted by reason, with a few example URLs for each.
| Reason | Meaning |
|---|---|
limit | The crawl had already queued limit pages. |
depth | Deeper than max_depth. |
out-of-scope | Above the start URL’s path. Set crawl_entire_domain to follow these. |
excluded | Matched exclude_paths. |
not-included | Matched none of include_paths. |
robots | Disallowed by the site’s robots.txt. |
off-site | On another site or a subdomain you did not allow. |
file | A file rather than a page — an image, an archive, a stylesheet. |
invalid | Not a URL the API will fetch, such as a private address. |
blocklist | On your key’s blocklist. |
duplicate | A page the crawl already had, under another URL or through a redirect. |
Example request
cURL
curl "https://api.prefetch.io/crawl/crw_9f2c1a7b4e0d4c3a8b1e5f7d2c9a0b3e/errors" \
-H "X-API-Key: $PREFETCH_API_KEY"
Related
POST /crawl
Start a crawl.
Error handling
What each error code means, and which are worth retrying.
Authorizations
Your Prefetch API key. Obtain one from the dashboard.
Path Parameters
The crawl id returned by POST /crawl.
Example:
"crw_9f2c1a7b4e0d4c3a8b1e5f7d2c9a0b3e"
Query Parameters
Number of result pages to skip. Use the next link rather than building this by hand.
Required range:
x >= 0Number of crawled pages to return per request.
Required range:
1 <= x <= 100