FlyCrawl Developer Integration Hub

# Direct Web-to-Vector Pipeline into Pinecone / Qdrant
from pinecone import Pinecone
from flycrawl import FlyCrawlApp

pc = Pinecone(api_key="YOUR_PINECONE_KEY")
index = pc.Index("flycrawl-rag")

fly = FlyCrawlApp(api_key="fly_live_your_key")

# Single call scrapes, chunks, normalizes, and embeds into 1536-dim vectors:
result = fly.scrape_url("https://docs.flycrawl.net/ai", params={"formats": ["embeddings", "markdown"]})

# Direct 1-line Upsert:
index.upsert(vectors=result["upsert_request"]["vectors"], namespace="production")
print(f"Successfully ingested {len(result['upsert_request']['vectors'])} vectors ready for semantic search!")
            

curl -X POST https://api.flycrawl.net/api/v1/scrape \
  -H "Authorization: Bearer fly_live_your_api_key" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com", "formats": ["embeddings", "markdown"]}'
            

# Official Python OpenAI Library Integration
from openai import OpenAI

client = OpenAI(
    base_url="https://flycrawl.net/v1",
    api_key="fly_live_your_api_key"
)

# Direct standalone embedding generation (Native 1536, Native Large 3072, or OpenAI)
response = client.embeddings.create(
    model="native-large",  # or "native", "text-embedding-3-small", "text-embedding-3-large"
    input=[
        "First document chunk for semantic retrieval",
        "Second paragraph ready for Pinecone / Qdrant"
    ]
)

for item in response.data:
    print(f"Vector {item.index}: {len(item.embedding)} dimensions generated!")
            

# pip install flycrawl
from flycrawl import FlyCrawlApp

app = FlyCrawlApp(api_key="fly_live_your_api_key")

# 1. Scrape with Fit-Markdown & 24h Smart Cache
doc = app.scrape_url("https://example.com", params={"format": "markdown", "maxAge": 86400})
print(doc["content"])

# 2. Fast Site Mapping (<2s)
sitemap = app.map_url("https://example.com", params={"limit": 1000})
print(f"Discovered {sitemap['total_links']} URLs")

# 3. Live Web Search & Auto-Scrape
search_results = app.search("latest LLM reasoning benchmarks", limit=5)
for res in search_results["results"]:
    print(res["title"], res["url"])
            

// npm install flycrawl
import { FlyCrawlApp } from 'flycrawl';

const app = new FlyCrawlApp({ apiKey: 'fly_live_your_api_key' });

// 1. Scrape with browser actions
const res = await app.scrapeUrl('https://example.com', {
  format: 'markdown',
  actions: [{ type: 'wait', milliseconds: 1000 }]
});
console.log(res.content);

// 2. Structured JSON Extract
const data = await app.extract('https://example.com/pricing', {
  prompt: 'Extract all pricing tiers and features'
});
console.log(data.data);
            

from flycrawl import FlyCrawlLoader
from langchain_text_splitters import MarkdownHeaderTextSplitter

# Load directly as clean LangChain Documents
loader = FlyCrawlLoader(
    url="https://example.com",
    api_key="fly_live_your_key",
    mode="scrape"
)
docs = loader.load()

splitter = MarkdownHeaderTextSplitter(headers_to_split_on=[("#", "H1"), ("##", "H2")])
chunks = splitter.split_text(docs[0].page_content)
print(f"Ready for vector ingestion: {len(chunks)} chunks")
            

from flycrawl import FlyCrawlReader
from llama_index.core import VectorStoreIndex

reader = FlyCrawlReader(api_key="fly_live_your_key")
documents = reader.load_data(url="https://example.com")

index = VectorStoreIndex.from_documents(documents)
query_engine = index.as_query_engine()
response = query_engine.query("What is this website about?")
print(response)
            

curl -X POST https://api.flycrawl.net/api/v1/map \
  -H "Authorization: Bearer fly_live_your_api_key" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com", "limit": 1000}'
            

curl -X POST https://api.flycrawl.net/api/v1/search \
  -H "Authorization: Bearer fly_live_your_api_key" \
  -H "Content-Type: application/json" \
  -d '{"query": "best vector databases comparison", "limit": 5}'