75 lines
2.4 KiB
PHP
75 lines
2.4 KiB
PHP
|
<?php
|
||
|
|
namespace AfFulltext;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Plain HTTP GET via cURL.
|
||
|
|
*
|
||
|
|
* Used by the CLI harness. Inside tt-rss the `direct` backend is TtrssFetcher
|
||
|
|
* instead, so article fetches keep tt-rss's own SSRF guard (UrlHelper::validate).
|
||
|
|
*/
|
||
|
|
final class CurlFetcher implements Fetcher {
|
||
|
|
public function __construct(
|
||
|
|
private readonly int $timeout = 20,
|
||
|
|
private readonly string $user_agent = 'Mozilla/5.0 (compatible; af_fulltext/1.0; +https://tt-rss.org/)',
|
||
|
|
) {}
|
||
|
|
|
||
|
|
public function name(): string { return 'direct'; }
|
||
|
|
|
||
|
|
public function fetch(string $url, array $headers = []): FetchResult {
|
||
|
|
$ch = curl_init();
|
||
|
|
|
||
|
|
$hdr = [];
|
||
|
|
foreach ($headers as $k => $v) $hdr[] = "$k: $v";
|
||
|
|
|
||
|
|
curl_setopt_array($ch, [
|
||
|
|
CURLOPT_URL => $url,
|
||
|
|
CURLOPT_RETURNTRANSFER => true,
|
||
|
|
CURLOPT_FOLLOWLOCATION => true,
|
||
|
|
CURLOPT_MAXREDIRS => 8,
|
||
|
|
CURLOPT_TIMEOUT => $this->timeout,
|
||
|
|
CURLOPT_CONNECTTIMEOUT => 10,
|
||
|
|
CURLOPT_ENCODING => '',
|
||
|
|
CURLOPT_USERAGENT => $headers['user-agent'] ?? $this->user_agent,
|
||
|
|
CURLOPT_HTTPHEADER => $hdr,
|
||
|
|
]);
|
||
|
|
|
||
|
|
$body = curl_exec($ch);
|
||
|
|
$err = curl_error($ch);
|
||
|
|
$status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||
|
|
$effective = (string) curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
|
||
|
|
$content_type = (string) curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
|
||
|
|
curl_close($ch);
|
||
|
|
|
||
|
|
if ($err !== '' || !is_string($body))
|
||
|
|
return new FetchResult(error: $err !== '' ? $err : 'empty response', status: $status, effective_url: $effective ?: $url);
|
||
|
|
|
||
|
|
if ($status >= 400)
|
||
|
|
return new FetchResult(error: "HTTP $status", status: $status, effective_url: $effective ?: $url);
|
||
|
|
|
||
|
|
return new FetchResult(
|
||
|
|
html: self::to_utf8($body, $content_type),
|
||
|
|
effective_url: $effective ?: $url,
|
||
|
|
status: $status,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Normalise to UTF-8. Several of these feeds are Polish, and a page served as
|
||
|
|
* ISO-8859-2 that is treated as UTF-8 loses every accented character.
|
||
|
|
*/
|
||
|
|
public static function to_utf8(string $body, string $content_type): string {
|
||
|
|
$charset = '';
|
||
|
|
|
||
|
|
if (preg_match('/charset=["\']?([\w-]+)/i', $content_type, $m)) $charset = $m[1];
|
||
|
|
|
||
|
|
if ($charset === '' && preg_match('/<meta[^>]+charset=["\']?([\w-]+)/i', substr($body, 0, 4096), $m))
|
||
|
|
$charset = $m[1];
|
||
|
|
|
||
|
|
if ($charset === '' || preg_match('/^utf-?8$/i', $charset)) return $body;
|
||
|
|
|
||
|
|
$converted = @mb_convert_encoding($body, 'UTF-8', $charset);
|
||
|
|
|
||
|
|
return is_string($converted) && $converted !== '' ? $converted : $body;
|
||
|
|
}
|
||
|
|
}
|