Skip to main content
GET
/
integrations
/
web_crawler
/
index
JavaScript
import Hyperspell from '@hyperspell/hyperspell';

const client = new Hyperspell({
  apiKey: process.env['HYPERSPELL_API_KEY'], // This is the default and can be omitted
});

const response = await client.integrations.webCrawler.index({ url: 'url' });

console.log(response.resource_id);
import os
from hyperspell import Hyperspell

client = Hyperspell(
api_key=os.environ.get("HYPERSPELL_API_KEY"), # This is the default and can be omitted
)
response = client.integrations.web_crawler.index(
url="url",
)
print(response.resource_id)
package main

import (
"context"
"fmt"

"github.com/hyperspell/hyperspell-go"
"github.com/hyperspell/hyperspell-go/option"
)

func main() {
client := hyperspell.NewClient(
option.WithAPIKey("My API Key"),
)
response, err := client.Integrations.WebCrawler.Index(context.TODO(), hyperspell.IntegrationWebCrawlerIndexParams{
URL: "url",
})
if err != nil {
panic(err.Error())
}
fmt.Printf("%+v\n", response.ResourceID)
}
hyperspell integrations:web-crawler index \
--api-key 'My API Key' \
--url url
curl --request GET \
--url https://api.hyperspell.com/integrations/web_crawler/index \
--header 'Authorization: Bearer <token>'
<?php

$curl = curl_init();

curl_setopt_array($curl, [
CURLOPT_URL => "https://api.hyperspell.com/integrations/web_crawler/index",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
HttpResponse<String> response = Unirest.get("https://api.hyperspell.com/integrations/web_crawler/index")
.header("Authorization", "Bearer <token>")
.asString();
require 'uri'
require 'net/http'

url = URI("https://api.hyperspell.com/integrations/web_crawler/index")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body
{
  "resource_id": "<string>"
}
{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}

Authorizations

Authorization
string
header
required

API Key or JWT User Token. If using an API Key, set the X-As-User header to act as a specific user. A JWT User Token is always scoped to a specific user.

Query Parameters

url
string
required

The base URL of the website to crawl

max_depth
integer
default:2

Maximum depth of links to follow during crawling

Required range: 0 <= x <= 5
limit
integer
default:20

Maximum number of pages to crawl in total

Required range: 1 <= x <= 100

Response

Successful Response

source
enum<string>
required
Available options:
reddit,
notion,
slack,
google_calendar,
google_mail,
box,
dropbox,
github,
google_drive,
vault,
web_crawler,
trace,
microsoft_teams,
gmail_actions,
granola,
fathom,
fireflies,
linear,
hubspot,
salesforce,
coda,
lightfield,
gong,
pylon,
clickup
resource_id
string
required
status
enum<string>
required
Available options:
pending,
processing,
completed,
failed,
pending_review,
skipped