> ## Documentation Index
> Fetch the complete documentation index at: https://docs.hyperspell.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Crawl a website for indexed search

> Recursively crawl a website to make it available for indexed search.



## OpenAPI

````yaml https://app.stainlessapi.com/api/spec/documented/hyperspell.yaml get /integrations/web_crawler/index
openapi: 3.1.0
info:
  title: Hyperspell API
  summary: >-
    Hyperspell is the memory layer for AI apps and agents. Through the API, you
    can add memories, connect data sources to index them in real-time, and
    search memories with our natural language query engine.
  termsOfService: https://hyperspell.com/blog/tos
  contact:
    name: Hyperspell
    url: https://hyperspell.com/
    email: hello@hyperspell.com
  version: 0.32.1
servers:
  - url: https://api.hyperspell.com
    description: Production
security:
  - APIKey: []
    AsUser: []
paths:
  /integrations/web_crawler/index:
    get:
      tags:
        - Integrations
        - Integrations
      summary: Crawl a website for indexed search
      description: Recursively crawl a website to make it available for indexed search.
      operationId: crawl_website_integrations_web_crawler_index_get
      parameters:
        - name: url
          in: query
          required: true
          schema:
            type: string
            description: The base URL of the website to crawl
            title: Url
          description: The base URL of the website to crawl
        - name: max_depth
          in: query
          required: false
          schema:
            type: integer
            maximum: 5
            minimum: 0
            description: Maximum depth of links to follow during crawling
            default: 2
            title: Max Depth
          description: Maximum depth of links to follow during crawling
        - name: limit
          in: query
          required: false
          schema:
            type: integer
            maximum: 100
            minimum: 1
            description: Maximum number of pages to crawl in total
            default: 20
            title: Limit
          description: Maximum number of pages to crawl in total
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ResourceStatusResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - APIKey: []
        - AsUser: []
      x-codeSamples:
        - lang: JavaScript
          source: >-
            import Hyperspell from '@hyperspell/hyperspell';


            const client = new Hyperspell({
              apiKey: process.env['HYPERSPELL_API_KEY'], // This is the default and can be omitted
            });


            const response = await client.integrations.webCrawler.index({ url:
            'url' });


            console.log(response.resource_id);
        - lang: Python
          source: |-
            import os
            from hyperspell import Hyperspell

            client = Hyperspell(
                api_key=os.environ.get("HYPERSPELL_API_KEY"),  # This is the default and can be omitted
            )
            response = client.integrations.web_crawler.index(
                url="url",
            )
            print(response.resource_id)
        - lang: Go
          source: "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/hyperspell/hyperspell-go\"\n\t\"github.com/hyperspell/hyperspell-go/option\"\n)\n\nfunc main() {\n\tclient := hyperspell.NewClient(\n\t\toption.WithAPIKey(\"My API Key\"),\n\t)\n\tresponse, err := client.Integrations.WebCrawler.Index(context.TODO(), hyperspell.IntegrationWebCrawlerIndexParams{\n\t\tURL: \"url\",\n\t})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\tfmt.Printf(\"%+v\\n\", response.ResourceID)\n}\n"
        - lang: CLI
          source: |-
            hyperspell integrations:web-crawler index \
              --api-key 'My API Key' \
              --url url
components:
  schemas:
    ResourceStatusResponse:
      properties:
        source:
          $ref: '#/components/schemas/DocumentProviders'
        resource_id:
          type: string
          title: Resource Id
        status:
          $ref: '#/components/schemas/DocumentStatus'
      type: object
      required:
        - source
        - resource_id
        - status
      title: ResourceStatusResponse
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    DocumentProviders:
      type: string
      enum:
        - reddit
        - notion
        - slack
        - google_calendar
        - google_mail
        - box
        - dropbox
        - github
        - google_drive
        - vault
        - web_crawler
        - trace
        - microsoft_teams
        - gmail_actions
        - granola
        - fathom
        - fireflies
        - linear
        - hubspot
        - salesforce
        - coda
        - lightfield
        - gong
        - pylon
      title: DocumentProviders
    DocumentStatus:
      type: string
      enum:
        - pending
        - processing
        - completed
        - failed
        - pending_review
        - skipped
      title: DocumentStatus
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
  securitySchemes:
    APIKey:
      type: http
      description: >-
        API Key or JWT User Token. If using an API Key, set the X-As-User header
        to act as a specific user. A JWT User Token is always scoped to a
        specific user.
      scheme: bearer
      bearerFormat: Bearer <token>
    AsUser:
      type: apiKey
      description: >-
        Optionally set this header to act as a specific user when using an API
        Key, equivalent to first exchanging the API Key for a User Token
      in: header
      name: X-As-User

````