Crawl Web Pages
curl --request POST \
--url https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website \
--header 'Content-Type: application/json' \
--data '
{
"page_urls": [
"https://example.com/faq",
"https://example.com/pricing"
]
}
'import requests
url = "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website"
payload = { "page_urls": ["https://example.com/faq", "https://example.com/pricing"] }
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({page_urls: ['https://example.com/faq', 'https://example.com/pricing']})
};
fetch('https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'page_urls' => [
'https://example.com/faq',
'https://example.com/pricing'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website"
payload := strings.NewReader("{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website")
.header("Content-Type", "application/json")
.body("{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"document_id": "<string>",
"status": "processing",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}{
"error": "<string>",
"message": "<string>"
}Knowledge Bases
Crawl Web Pages
Crawls the given pages for asynchronous ingestion.
One call creates one document covering every URL in the list, not one document per URL. Like file upload, a 202 means queued, not indexed - poll the document status endpoint until it reports ‘completed’.
There is no document update endpoint. Re-crawling the same URLs creates a second document while the first remains; to refresh, delete and re-crawl.
POST
/
api
/
v1
/
knowledge-bases
/
{kb_id}
/
documents
/
website
Crawl Web Pages
curl --request POST \
--url https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website \
--header 'Content-Type: application/json' \
--data '
{
"page_urls": [
"https://example.com/faq",
"https://example.com/pricing"
]
}
'import requests
url = "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website"
payload = { "page_urls": ["https://example.com/faq", "https://example.com/pricing"] }
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({page_urls: ['https://example.com/faq', 'https://example.com/pricing']})
};
fetch('https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'page_urls' => [
'https://example.com/faq',
'https://example.com/pricing'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website"
payload := strings.NewReader("{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website")
.header("Content-Type", "application/json")
.body("{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.verbex.ai/api/v1/knowledge-bases/{kb_id}/documents/website")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"page_urls\": [\n \"https://example.com/faq\",\n \"https://example.com/pricing\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"document_id": "<string>",
"status": "processing",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"error": "<string>",
"message": "<string>"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}{
"error": "<string>",
"message": "<string>"
}Body
application/json
Response
Crawl accepted and queued. Not yet searchable.
Acknowledgement that a document was accepted for asynchronous ingestion.

