81 lines
2.7 KiB
PHP
81 lines
2.7 KiB
PHP
|
<?php
|
||
|
|
namespace AfFulltext;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Fetch through a Firecrawl instance so the page arrives with its JavaScript
|
||
|
|
* already run.
|
||
|
|
*
|
||
|
|
* We ask for `rawHtml` on purpose. Firecrawl's own `onlyMainContent` extraction
|
||
|
|
* was measured against these feeds and is far too coarse -- on a Cloudflare blog
|
||
|
|
* post it returned 73KB still containing skip-links, analytics markup and the
|
||
|
|
* language-picker footer. Firecrawl is the better *renderer*; the rule engine in
|
||
|
|
* this plugin is the better *extractor*. So: browser here, extraction ours.
|
||
|
|
*/
|
||
|
|
final class FirecrawlFetcher implements Fetcher {
|
||
|
|
public function __construct(
|
||
|
|
private readonly string $endpoint,
|
||
|
|
private readonly int $timeout = 60,
|
||
|
|
private readonly ?string $api_key = null,
|
||
|
|
) {}
|
||
|
|
|
||
|
|
public function name(): string { return 'firecrawl'; }
|
||
|
|
|
||
|
|
public function fetch(string $url, array $headers = []): FetchResult {
|
||
|
|
$payload = [
|
||
|
|
'url' => $url,
|
||
|
|
'formats' => ['rawHtml'],
|
||
|
|
// Firecrawl's own boilerplate stripping is deliberately off; see above.
|
||
|
|
'onlyMainContent' => false,
|
||
|
|
'timeout' => $this->timeout * 1000,
|
||
|
|
];
|
||
|
|
|
||
|
|
if ($headers) $payload['headers'] = $headers;
|
||
|
|
|
||
|
|
$request_headers = ['Content-Type: application/json'];
|
||
|
|
if ($this->api_key) $request_headers[] = 'Authorization: Bearer ' . $this->api_key;
|
||
|
|
|
||
|
|
$ch = curl_init();
|
||
|
|
curl_setopt_array($ch, [
|
||
|
|
CURLOPT_URL => rtrim($this->endpoint, '/') . '/v1/scrape',
|
||
|
|
CURLOPT_POST => true,
|
||
|
|
CURLOPT_POSTFIELDS => json_encode($payload, JSON_UNESCAPED_SLASHES),
|
||
|
|
CURLOPT_HTTPHEADER => $request_headers,
|
||
|
|
CURLOPT_RETURNTRANSFER => true,
|
||
|
|
// The browser render dominates; allow slack over the scrape timeout.
|
||
|
|
CURLOPT_TIMEOUT => $this->timeout + 15,
|
||
|
|
CURLOPT_CONNECTTIMEOUT => 10,
|
||
|
|
]);
|
||
|
|
|
||
|
|
$body = curl_exec($ch);
|
||
|
|
$err = curl_error($ch);
|
||
|
|
$status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||
|
|
curl_close($ch);
|
||
|
|
|
||
|
|
if ($err !== '' || !is_string($body))
|
||
|
|
return new FetchResult(error: 'firecrawl: ' . ($err !== '' ? $err : 'empty response'), status: $status);
|
||
|
|
|
||
|
|
$json = json_decode($body, true);
|
||
|
|
if (!is_array($json))
|
||
|
|
return new FetchResult(error: 'firecrawl: unparseable response', status: $status);
|
||
|
|
|
||
|
|
if (empty($json['success'])) {
|
||
|
|
$msg = is_string($json['error'] ?? null) ? $json['error'] : "HTTP $status";
|
||
|
|
return new FetchResult(error: "firecrawl: $msg", status: $status);
|
||
|
|
}
|
||
|
|
|
||
|
|
$data = $json['data'] ?? [];
|
||
|
|
$html = $data['rawHtml'] ?? $data['html'] ?? '';
|
||
|
|
|
||
|
|
if (!is_string($html) || $html === '')
|
||
|
|
return new FetchResult(error: 'firecrawl: no html in response', status: $status);
|
||
|
|
|
||
|
|
$effective = $data['metadata']['sourceURL'] ?? $data['metadata']['url'] ?? $url;
|
||
|
|
|
||
|
|
return new FetchResult(
|
||
|
|
html: $html,
|
||
|
|
effective_url: is_string($effective) ? $effective : $url,
|
||
|
|
status: $status,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|