be8205657e
- Fixed Firecrawl webhook payloads to ensure proper data handling and delivery. - Removed unused Robin AI code to improve codebase cleanliness and maintainability. - Implement authentication for the Firecrawl endpoint to improve security. A key is generated to secure the webhook URLs from FireCrawl. --------- Co-authored-by: Pranav <pranavrajs@gmail.com>
41 lines
932 B
Ruby
41 lines
932 B
Ruby
class Captain::Tools::FirecrawlService
|
|
def initialize
|
|
@api_key = InstallationConfig.find_by!(name: 'CAPTAIN_FIRECRAWL_API_KEY').value
|
|
raise 'Missing API key' if @api_key.nil?
|
|
end
|
|
|
|
def perform(url, webhook_url, crawl_limit = 10)
|
|
HTTParty.post(
|
|
'https://api.firecrawl.dev/v1/crawl',
|
|
body: crawl_payload(url, webhook_url, crawl_limit),
|
|
headers: headers
|
|
)
|
|
rescue StandardError => e
|
|
raise "Failed to crawl URL: #{e.message}"
|
|
end
|
|
|
|
private
|
|
|
|
def crawl_payload(url, webhook_url, crawl_limit)
|
|
{
|
|
url: url,
|
|
maxDepth: 50,
|
|
ignoreSitemap: false,
|
|
limit: crawl_limit,
|
|
webhook: webhook_url,
|
|
scrapeOptions: {
|
|
onlyMainContent: false,
|
|
formats: ['markdown'],
|
|
excludeTags: ['iframe']
|
|
}
|
|
}.to_json
|
|
end
|
|
|
|
def headers
|
|
{
|
|
'Authorization' => "Bearer #{@api_key}",
|
|
'Content-Type' => 'application/json'
|
|
}
|
|
end
|
|
end
|