firecrawl-data-handling

# Firecrawl Data Handling ## Overview Process scraped web content from Firecrawl pipelines. Covers markdown cleaning, structured data extraction with Zod validation, content deduplication, chunking for LLM/RAG, and storage patterns for crawled content. ## Instructions ### Step 1: Content Cleaning ```typescript import FirecrawlApp from "@mendable/firecrawl-js"; const firecrawl = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY!, }); // Scrape with clean output settings async function scrapeClean(url: string) { const result = await firecrawl.scrapeUrl(url, { formats: ["markdown"], onlyMainContent: true, // strips nav, footer, sidebar excludeTags: ["script", "style", "nav", "footer", "iframe"], waitFor: 2000, }); return { url: result.metadata?.sourceURL || url, title: result.metadata?.title || "", markdown: cleanMarkdown(result.markdown || ""), scrapedAt: new Date().toISOString(), }; } function cleanMarkdown(md: string): string { return md .replace(/\n{3,}/g, "\n\n") // collapse multiple newlines .replace(/\[.*?\]\(javascript:.*?\)/g, "") // remove JS links .replace(/!\[.*?\]\(data:.*?\)/g, "") // remove inline data URIs .replace(/<!--[\s\S]*?-->/g, "") // remove HTML comments .replace(/<script[\s\S]*?<\/script>/gi, "") // remove script tags .trim(); } ``` ### Step 2: Structured Extraction with Validation ```typescript import { z } from "zod"...