af_fulltext: rule-driven extraction with a pluggable renderer
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs go back to being real feed URLs and extraction happens inside tt-rss, using the same ftr-site-config rules the proxy used. The engine in lib/ has no tt-rss dependencies, so rules can be developed and audited from the command line; init.php is a thin adapter over it. Two findings from measuring the real subscription first, both of which shaped the design: - Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of chrome on a Cloudflare post), but it is an excellent renderer. So it is used for rawHtml only and the rule engine does the extraction. - A body rule that stops matching after a redesign falls through to Readability and still produces a plausible article, so the breakage is invisible. Every extraction now records which rule matched and whether it fell back; auditing the 34 live feeds surfaced five community rules that match nothing. Custom rules included for the sites that needed them, including three comics where the article is an image and text-scoring extractors return the wrong thing or nothing at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
This commit is contained in:
Michal
74
lib/CurlFetcher.php
Normal file
74
lib/CurlFetcher.php
Normal file
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* Plain HTTP GET via cURL.
|
||||
*
|
||||
* Used by the CLI harness. Inside tt-rss the `direct` backend is TtrssFetcher
|
||||
* instead, so article fetches keep tt-rss's own SSRF guard (UrlHelper::validate).
|
||||
*/
|
||||
final class CurlFetcher implements Fetcher {
|
||||
public function __construct(
|
||||
private readonly int $timeout = 20,
|
||||
private readonly string $user_agent = 'Mozilla/5.0 (compatible; af_fulltext/1.0; +https://tt-rss.org/)',
|
||||
) {}
|
||||
|
||||
public function name(): string { return 'direct'; }
|
||||
|
||||
public function fetch(string $url, array $headers = []): FetchResult {
|
||||
$ch = curl_init();
|
||||
|
||||
$hdr = [];
|
||||
foreach ($headers as $k => $v) $hdr[] = "$k: $v";
|
||||
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_URL => $url,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
CURLOPT_MAXREDIRS => 8,
|
||||
CURLOPT_TIMEOUT => $this->timeout,
|
||||
CURLOPT_CONNECTTIMEOUT => 10,
|
||||
CURLOPT_ENCODING => '',
|
||||
CURLOPT_USERAGENT => $headers['user-agent'] ?? $this->user_agent,
|
||||
CURLOPT_HTTPHEADER => $hdr,
|
||||
]);
|
||||
|
||||
$body = curl_exec($ch);
|
||||
$err = curl_error($ch);
|
||||
$status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||||
$effective = (string) curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
|
||||
$content_type = (string) curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($err !== '' || !is_string($body))
|
||||
return new FetchResult(error: $err !== '' ? $err : 'empty response', status: $status, effective_url: $effective ?: $url);
|
||||
|
||||
if ($status >= 400)
|
||||
return new FetchResult(error: "HTTP $status", status: $status, effective_url: $effective ?: $url);
|
||||
|
||||
return new FetchResult(
|
||||
html: self::to_utf8($body, $content_type),
|
||||
effective_url: $effective ?: $url,
|
||||
status: $status,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalise to UTF-8. Several of these feeds are Polish, and a page served as
|
||||
* ISO-8859-2 that is treated as UTF-8 loses every accented character.
|
||||
*/
|
||||
public static function to_utf8(string $body, string $content_type): string {
|
||||
$charset = '';
|
||||
|
||||
if (preg_match('/charset=["\']?([\w-]+)/i', $content_type, $m)) $charset = $m[1];
|
||||
|
||||
if ($charset === '' && preg_match('/<meta[^>]+charset=["\']?([\w-]+)/i', substr($body, 0, 4096), $m))
|
||||
$charset = $m[1];
|
||||
|
||||
if ($charset === '' || preg_match('/^utf-?8$/i', $charset)) return $body;
|
||||
|
||||
$converted = @mb_convert_encoding($body, 'UTF-8', $charset);
|
||||
|
||||
return is_string($converted) && $converted !== '' ? $converted : $body;
|
||||
}
|
||||
}
|
||||
57
lib/ExtractResult.php
Normal file
57
lib/ExtractResult.php
Normal file
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* The outcome of one extraction, diagnostics included.
|
||||
*
|
||||
* The diagnostics are the point. A site redesign turns a `body:` rule into a
|
||||
* silent no-op that falls through to Readability, and the article still looks
|
||||
* plausible -- that is how a broken webcomic rule went unnoticed here for years.
|
||||
* Every consumer of this class can see exactly which rule ran and whether it
|
||||
* actually matched anything.
|
||||
*/
|
||||
final class ExtractResult {
|
||||
public string $html = '';
|
||||
public ?string $title = null;
|
||||
public ?string $author = null;
|
||||
public ?string $date = null;
|
||||
|
||||
public string $backend = '';
|
||||
public string $effective_url = '';
|
||||
|
||||
/** Rule files that contributed, nearest first. @var string[] */
|
||||
public array $rule_sources = [];
|
||||
|
||||
/** True when a `body:` rule existed and selected at least one node. */
|
||||
public bool $rule_matched = false;
|
||||
|
||||
/** True when the content came from Readability rather than a rule. */
|
||||
public bool $fell_back = false;
|
||||
|
||||
/** True when a rule existed but matched nothing -- the stale-rule signal. */
|
||||
public bool $rule_stale = false;
|
||||
|
||||
public int $pages = 1;
|
||||
public int $timing_ms = 0;
|
||||
|
||||
/** @var string[] */
|
||||
public array $errors = [];
|
||||
|
||||
public function ok(): bool {
|
||||
return $this->html !== '';
|
||||
}
|
||||
|
||||
/** One-line summary for logs and the prefs UI. */
|
||||
public function summary(): string {
|
||||
$rule = $this->rule_sources ? basename($this->rule_sources[0]) : 'none';
|
||||
|
||||
$how = match (true) {
|
||||
$this->rule_matched => "rule=$rule",
|
||||
$this->rule_stale => "rule=$rule STALE->readability",
|
||||
default => 'readability',
|
||||
};
|
||||
|
||||
return sprintf('%s %s %db %dms%s', $this->backend, $how, strlen($this->html),
|
||||
$this->timing_ms, $this->pages > 1 ? " pages={$this->pages}" : '');
|
||||
}
|
||||
}
|
||||
361
lib/Extractor.php
Normal file
361
lib/Extractor.php
Normal file
@@ -0,0 +1,361 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
use fivefilters\Readability\Configuration;
|
||||
use fivefilters\Readability\Readability;
|
||||
|
||||
/**
|
||||
* Turns a URL into clean article HTML.
|
||||
*
|
||||
* Rule-driven where a site config exists, Readability where one does not. The
|
||||
* rule format is FiveFilters' own, so the ~2000 community rules in
|
||||
* fivefilters/ftr-site-config apply unmodified and anything written here can go
|
||||
* back upstream as a pull request.
|
||||
*/
|
||||
final class Extractor {
|
||||
private const MAX_PAGES = 5;
|
||||
|
||||
public function __construct(
|
||||
private readonly RuleSet $rules,
|
||||
private readonly int $max_bytes = 4_000_000,
|
||||
) {}
|
||||
|
||||
public function extract(string $url, Fetcher $fetcher): ExtractResult {
|
||||
$started = microtime(true);
|
||||
|
||||
$result = new ExtractResult();
|
||||
$result->backend = $fetcher->name();
|
||||
$result->effective_url = $url;
|
||||
|
||||
$host = strtolower((string) parse_url($url, PHP_URL_HOST));
|
||||
$rule = $this->rules->find($host);
|
||||
|
||||
if ($rule) $result->rule_sources = $rule->sources;
|
||||
|
||||
$fetched = $fetcher->fetch($url, $rule?->http_headers ?? []);
|
||||
|
||||
if (!$fetched->ok()) {
|
||||
$result->errors[] = $fetched->error ?? 'fetch failed';
|
||||
$result->timing_ms = (int) round((microtime(true) - $started) * 1000);
|
||||
return $result;
|
||||
}
|
||||
|
||||
if (strlen($fetched->html) > $this->max_bytes)
|
||||
$result->errors[] = sprintf('page is %dKB, truncating parse', strlen($fetched->html) / 1024);
|
||||
|
||||
$result->effective_url = $fetched->effective_url ?: $url;
|
||||
|
||||
$doc = Html::parse(substr($fetched->html, 0, $this->max_bytes));
|
||||
$base = Html::base_url($doc, $result->effective_url);
|
||||
|
||||
// A print/single-page view, where one exists, is both cleaner and cheaper
|
||||
// to extract than following next_page_link through N requests.
|
||||
if ($rule && $rule->single_page_links) {
|
||||
$single = $this->first_url($doc, $rule->single_page_links, $base);
|
||||
|
||||
if ($single && $single !== $result->effective_url) {
|
||||
$again = $fetcher->fetch($single, $rule->http_headers);
|
||||
|
||||
if ($again->ok()) {
|
||||
$doc = Html::parse(substr($again->html, 0, $this->max_bytes));
|
||||
$result->effective_url = $again->effective_url ?: $single;
|
||||
$base = Html::base_url($doc, $result->effective_url);
|
||||
} else {
|
||||
$result->errors[] = 'single_page_link: ' . ($again->error ?? 'failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$html = $this->extract_from_doc($doc, $base, $rule, $result);
|
||||
|
||||
// Multi-page articles: keep appending until the chain ends or we hit the cap.
|
||||
if ($rule && $rule->next_page_links) {
|
||||
$seen = [$result->effective_url => true];
|
||||
$current = $doc;
|
||||
$current_base = $base;
|
||||
|
||||
while ($result->pages < self::MAX_PAGES) {
|
||||
$next = $this->first_url($current, $rule->next_page_links, $current_base);
|
||||
if (!$next || isset($seen[$next])) break;
|
||||
|
||||
$seen[$next] = true;
|
||||
$page = $fetcher->fetch($next, $rule->http_headers);
|
||||
|
||||
if (!$page->ok()) { $result->errors[] = 'next_page_link: ' . ($page->error ?? 'failed'); break; }
|
||||
|
||||
$current = Html::parse(substr($page->html, 0, $this->max_bytes));
|
||||
$current_base = Html::base_url($current, $page->effective_url ?: $next);
|
||||
|
||||
$more = $this->extract_from_doc($current, $current_base, $rule, new ExtractResult());
|
||||
if ($more === '') break;
|
||||
|
||||
$html .= "\n" . $more;
|
||||
$result->pages++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($rule) {
|
||||
$html = $this->apply_replacements($html, $rule);
|
||||
$html = $this->apply_wrap_in($html, $rule);
|
||||
}
|
||||
|
||||
$result->html = trim($html);
|
||||
$result->timing_ms = (int) round((microtime(true) - $started) * 1000);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the rule pipeline over one parsed document.
|
||||
*
|
||||
* Order matters: lazy images are resolved and URLs made absolute before
|
||||
* anything is removed or selected, so every later step sees the same, final
|
||||
* attribute values.
|
||||
*/
|
||||
private function extract_from_doc(\DOMDocument $doc, string $base, ?Rule $rule, ExtractResult $result): string {
|
||||
Html::unlazy($doc);
|
||||
Html::absolutize($doc, $base);
|
||||
|
||||
$xpath = new \DOMXPath($doc);
|
||||
|
||||
if ($rule) {
|
||||
$this->apply_removals($doc, $xpath, $rule);
|
||||
$this->apply_dissolve($xpath, $rule);
|
||||
|
||||
$result->title ??= $this->first_string($xpath, $rule->titles);
|
||||
$result->author ??= $this->first_string($xpath, $rule->authors);
|
||||
$result->date ??= $this->first_string($xpath, $rule->dates);
|
||||
}
|
||||
|
||||
if ($rule && $rule->bodies) {
|
||||
$nodes = $this->select_body($xpath, $rule->bodies);
|
||||
|
||||
if ($nodes) {
|
||||
$result->rule_matched = true;
|
||||
|
||||
$out = '';
|
||||
foreach ($nodes as $node) {
|
||||
Html::sanitize($node);
|
||||
$out .= Html::outer_html($node) . "\n";
|
||||
}
|
||||
|
||||
return trim($out);
|
||||
}
|
||||
|
||||
// A rule exists and selected nothing. Almost always a site redesign.
|
||||
$result->rule_stale = true;
|
||||
$result->errors[] = 'body rule matched no nodes (' . implode(', ', $rule->bodies) . ')';
|
||||
|
||||
if (!$rule->autodetect_on_failure) return '';
|
||||
}
|
||||
|
||||
$result->fell_back = true;
|
||||
|
||||
return $this->readability($doc, $base, $result);
|
||||
}
|
||||
|
||||
/** @return \DOMNode[] */
|
||||
private function select_body(\DOMXPath $xpath, array $expressions): array {
|
||||
$nodes = [];
|
||||
|
||||
foreach ($expressions as $expr) {
|
||||
$found = @$xpath->query($expr);
|
||||
if ($found === false) continue;
|
||||
|
||||
foreach ($found as $node) {
|
||||
if (Html::text_length($node) > 0 || $this->has_media($node)) $nodes[] = $node;
|
||||
}
|
||||
}
|
||||
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
/**
|
||||
* A node with no text is not necessarily empty -- on a webcomic the entire
|
||||
* article is a single <img>, and discarding it for having no words is exactly
|
||||
* the bug this plugin exists to fix.
|
||||
*/
|
||||
private const MEDIA_TAGS = ['img', 'video', 'audio', 'iframe', 'picture', 'source', 'svg', 'embed', 'object'];
|
||||
|
||||
private function has_media(\DOMNode $node): bool {
|
||||
if (!$node instanceof \DOMElement && !$node instanceof \DOMDocument) return false;
|
||||
|
||||
// The node may BE the media. `body: //img[@id='strip']` is a perfectly
|
||||
// ordinary rule for an image-only comic, and only looking at descendants
|
||||
// throws that selection away.
|
||||
if ($node instanceof \DOMElement && in_array(strtolower($node->tagName), self::MEDIA_TAGS, true))
|
||||
return true;
|
||||
|
||||
$doc = $node instanceof \DOMDocument ? $node : $node->ownerDocument;
|
||||
if (!$doc) return false;
|
||||
|
||||
$query = implode(' | ', array_map(fn(string $t) => ".//$t", self::MEDIA_TAGS));
|
||||
$found = @(new \DOMXPath($doc))->query($query, $node);
|
||||
|
||||
return $found !== false && $found->length > 0;
|
||||
}
|
||||
|
||||
private function apply_removals(\DOMDocument $doc, \DOMXPath $xpath, Rule $rule): void {
|
||||
foreach ($rule->strips as $expr) {
|
||||
$found = @$xpath->query($expr);
|
||||
if ($found === false) continue;
|
||||
foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
|
||||
}
|
||||
|
||||
foreach ($rule->strip_id_or_class as $needle) {
|
||||
$q = $this->contains_lower('@id', $needle) . ' or ' . $this->contains_lower('@class', $needle);
|
||||
$found = @$xpath->query("//*[$q]");
|
||||
if ($found === false) continue;
|
||||
foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
|
||||
}
|
||||
|
||||
foreach ($rule->strip_image_src as $needle) {
|
||||
$found = @$xpath->query('//img[' . $this->contains_lower('@src', $needle) . ']');
|
||||
if ($found === false) continue;
|
||||
foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
|
||||
}
|
||||
}
|
||||
|
||||
/** Case-insensitive substring test, spelled out because XPath 1.0 has no lower-case(). */
|
||||
private function contains_lower(string $attr, string $needle): string {
|
||||
$upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
|
||||
$lower = 'abcdefghijklmnopqrstuvwxyz';
|
||||
$q = $this->xpath_literal(strtolower($needle));
|
||||
|
||||
return "contains(translate($attr, '$upper', '$lower'), $q)";
|
||||
}
|
||||
|
||||
/** Quote a string for XPath 1.0, which has no escape syntax. */
|
||||
private function xpath_literal(string $s): string {
|
||||
if (!str_contains($s, "'")) return "'$s'";
|
||||
if (!str_contains($s, '"')) return "\"$s\"";
|
||||
|
||||
return 'concat(' . implode(", \"'\", ", array_map(fn($p) => "'$p'", explode("'", $s))) . ')';
|
||||
}
|
||||
|
||||
/** Replace matched elements with their own children. */
|
||||
private function apply_dissolve(\DOMXPath $xpath, Rule $rule): void {
|
||||
foreach ($rule->dissolve as $expr) {
|
||||
$found = @$xpath->query($expr);
|
||||
if ($found === false) continue;
|
||||
|
||||
foreach (iterator_to_array($found) as $node) {
|
||||
$parent = $node->parentNode;
|
||||
if (!$parent) continue;
|
||||
|
||||
while ($node->firstChild) $parent->insertBefore($node->firstChild, $node);
|
||||
$parent->removeChild($node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Readability fallback.
|
||||
*
|
||||
* The engine emits notices on malformed pages (null property reads on
|
||||
* documents with no discernible body). Those must not reach output: inside
|
||||
* tt-rss this runs mid-request and a stray warning corrupts the response. So
|
||||
* diagnostics are captured into the result instead of being printed, and the
|
||||
* previous handler is always restored.
|
||||
*/
|
||||
private function readability(\DOMDocument $doc, string $base, ExtractResult $result): string {
|
||||
$config = new Configuration();
|
||||
$config->setOriginalURL($base);
|
||||
$config->setFixRelativeURLs(true);
|
||||
$config->setParser('html5');
|
||||
// Class names carry meaning for downstream styling and for anyone writing
|
||||
// a rule off the extracted output.
|
||||
$config->setKeepClasses(true);
|
||||
|
||||
$notices = [];
|
||||
|
||||
set_error_handler(function (int $no, string $msg) use (&$notices): bool {
|
||||
$notices[] = $msg;
|
||||
return true;
|
||||
});
|
||||
|
||||
try {
|
||||
$readability = new Readability($config);
|
||||
$readability->parse((string) $doc->saveHTML());
|
||||
|
||||
$result->title ??= $readability->getTitle();
|
||||
$result->author ??= $readability->getAuthor();
|
||||
|
||||
$content = (string) $readability->getContent();
|
||||
} catch (\Throwable $e) {
|
||||
$result->errors[] = 'readability: ' . $e->getMessage();
|
||||
$content = '';
|
||||
} finally {
|
||||
restore_error_handler();
|
||||
}
|
||||
|
||||
if ($notices)
|
||||
$result->errors[] = sprintf('readability emitted %d notice(s): %s',
|
||||
count($notices), $notices[0]);
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
private function apply_replacements(string $html, Rule $rule): string {
|
||||
foreach ($rule->find_strings as $i => $find) {
|
||||
$replace = $rule->replace_strings[$i] ?? '';
|
||||
if ($find !== '') $html = str_replace($find, $replace, $html);
|
||||
}
|
||||
|
||||
return $html;
|
||||
}
|
||||
|
||||
private function apply_wrap_in(string $html, Rule $rule): string {
|
||||
// wrap_in targets nodes in the source document; applying it to the
|
||||
// already-serialized result would need a reparse for little gain, so v1
|
||||
// only honours a document-wide wrapper.
|
||||
foreach ($rule->wrap_in as $expr => $spec) {
|
||||
if ($expr !== '//*' && $expr !== '/') continue;
|
||||
|
||||
[$tag, $class] = array_pad(explode('.', $spec, 2), 2, null);
|
||||
$tag = $tag ?: 'div';
|
||||
$attr = $class ? ' class="' . htmlspecialchars($class, ENT_QUOTES) . '"' : '';
|
||||
$html = "<$tag$attr>$html</$tag>";
|
||||
}
|
||||
|
||||
return $html;
|
||||
}
|
||||
|
||||
/** First non-empty string value produced by any of the expressions. */
|
||||
private function first_string(\DOMXPath $xpath, array $expressions): ?string {
|
||||
foreach ($expressions as $expr) {
|
||||
$value = @$xpath->evaluate($expr);
|
||||
|
||||
if (is_string($value) && trim($value) !== '') return trim($value);
|
||||
|
||||
if ($value instanceof \DOMNodeList && $value->length > 0) {
|
||||
$text = trim((string) $value->item(0)?->textContent);
|
||||
if ($text !== '') return $text;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** First absolute URL produced by any of the expressions. */
|
||||
private function first_url(\DOMDocument $doc, array $expressions, string $base): ?string {
|
||||
$xpath = new \DOMXPath($doc);
|
||||
|
||||
foreach ($expressions as $expr) {
|
||||
$value = @$xpath->evaluate($expr);
|
||||
$raw = null;
|
||||
|
||||
if (is_string($value) && trim($value) !== '') {
|
||||
$raw = trim($value);
|
||||
} elseif ($value instanceof \DOMNodeList && $value->length > 0) {
|
||||
$node = $value->item(0);
|
||||
$raw = $node instanceof \DOMElement ? ($node->getAttribute('href') ?: trim($node->textContent))
|
||||
: trim((string) $node?->nodeValue);
|
||||
}
|
||||
|
||||
if ($raw) return Html::resolve($raw, $base);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
29
lib/Fetcher.php
Normal file
29
lib/Fetcher.php
Normal file
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
final class FetchResult {
|
||||
public function __construct(
|
||||
public readonly string $html = '',
|
||||
public readonly string $effective_url = '',
|
||||
public readonly int $status = 0,
|
||||
public readonly ?string $error = null,
|
||||
) {}
|
||||
|
||||
public function ok(): bool {
|
||||
return $this->error === null && $this->html !== '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* How a page's HTML is obtained. Kept behind an interface so the extraction
|
||||
* pipeline is identical whether the markup came from a plain HTTP GET or from a
|
||||
* headless browser -- and so the engine can be exercised from the CLI without
|
||||
* tt-rss present.
|
||||
*/
|
||||
interface Fetcher {
|
||||
/** @param array<string,string> $headers extra request headers (lowercased names) */
|
||||
public function fetch(string $url, array $headers = []): FetchResult;
|
||||
|
||||
/** Short identifier used in logs and the prefs UI. */
|
||||
public function name(): string;
|
||||
}
|
||||
80
lib/FirecrawlFetcher.php
Normal file
80
lib/FirecrawlFetcher.php
Normal file
@@ -0,0 +1,80 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* Fetch through a Firecrawl instance so the page arrives with its JavaScript
|
||||
* already run.
|
||||
*
|
||||
* We ask for `rawHtml` on purpose. Firecrawl's own `onlyMainContent` extraction
|
||||
* was measured against these feeds and is far too coarse -- on a Cloudflare blog
|
||||
* post it returned 73KB still containing skip-links, analytics markup and the
|
||||
* language-picker footer. Firecrawl is the better *renderer*; the rule engine in
|
||||
* this plugin is the better *extractor*. So: browser here, extraction ours.
|
||||
*/
|
||||
final class FirecrawlFetcher implements Fetcher {
|
||||
public function __construct(
|
||||
private readonly string $endpoint,
|
||||
private readonly int $timeout = 60,
|
||||
private readonly ?string $api_key = null,
|
||||
) {}
|
||||
|
||||
public function name(): string { return 'firecrawl'; }
|
||||
|
||||
public function fetch(string $url, array $headers = []): FetchResult {
|
||||
$payload = [
|
||||
'url' => $url,
|
||||
'formats' => ['rawHtml'],
|
||||
// Firecrawl's own boilerplate stripping is deliberately off; see above.
|
||||
'onlyMainContent' => false,
|
||||
'timeout' => $this->timeout * 1000,
|
||||
];
|
||||
|
||||
if ($headers) $payload['headers'] = $headers;
|
||||
|
||||
$request_headers = ['Content-Type: application/json'];
|
||||
if ($this->api_key) $request_headers[] = 'Authorization: Bearer ' . $this->api_key;
|
||||
|
||||
$ch = curl_init();
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_URL => rtrim($this->endpoint, '/') . '/v1/scrape',
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode($payload, JSON_UNESCAPED_SLASHES),
|
||||
CURLOPT_HTTPHEADER => $request_headers,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
// The browser render dominates; allow slack over the scrape timeout.
|
||||
CURLOPT_TIMEOUT => $this->timeout + 15,
|
||||
CURLOPT_CONNECTTIMEOUT => 10,
|
||||
]);
|
||||
|
||||
$body = curl_exec($ch);
|
||||
$err = curl_error($ch);
|
||||
$status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($err !== '' || !is_string($body))
|
||||
return new FetchResult(error: 'firecrawl: ' . ($err !== '' ? $err : 'empty response'), status: $status);
|
||||
|
||||
$json = json_decode($body, true);
|
||||
if (!is_array($json))
|
||||
return new FetchResult(error: 'firecrawl: unparseable response', status: $status);
|
||||
|
||||
if (empty($json['success'])) {
|
||||
$msg = is_string($json['error'] ?? null) ? $json['error'] : "HTTP $status";
|
||||
return new FetchResult(error: "firecrawl: $msg", status: $status);
|
||||
}
|
||||
|
||||
$data = $json['data'] ?? [];
|
||||
$html = $data['rawHtml'] ?? $data['html'] ?? '';
|
||||
|
||||
if (!is_string($html) || $html === '')
|
||||
return new FetchResult(error: 'firecrawl: no html in response', status: $status);
|
||||
|
||||
$effective = $data['metadata']['sourceURL'] ?? $data['metadata']['url'] ?? $url;
|
||||
|
||||
return new FetchResult(
|
||||
html: $html,
|
||||
effective_url: is_string($effective) ? $effective : $url,
|
||||
status: $status,
|
||||
);
|
||||
}
|
||||
}
|
||||
259
lib/Html.php
Normal file
259
lib/Html.php
Normal file
@@ -0,0 +1,259 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
use Masterminds\HTML5;
|
||||
|
||||
/** DOM helpers shared by the extractor. No tt-rss dependencies. */
|
||||
final class Html {
|
||||
/**
|
||||
* Parse a document. Masterminds' HTML5 parser handles modern markup that
|
||||
* libxml mangles (unquoted attributes, <picture>, custom elements); libxml is
|
||||
* the fallback because it copes with badly broken pages that make the HTML5
|
||||
* parser give up.
|
||||
*/
|
||||
public static function parse(string $html): \DOMDocument {
|
||||
$html = self::strip_bom($html);
|
||||
|
||||
try {
|
||||
$doc = (new HTML5(['disable_html_ns' => true]))->loadHTML($html);
|
||||
if ($doc->documentElement !== null) return $doc;
|
||||
} catch (\Throwable) {
|
||||
// fall through
|
||||
}
|
||||
|
||||
$doc = new \DOMDocument();
|
||||
$prev = libxml_use_internal_errors(true);
|
||||
// Force UTF-8: libxml assumes ISO-8859-1 without a meta charset, which
|
||||
// silently mojibakes every non-ASCII article (most of these feeds are Polish).
|
||||
$doc->loadHTML('<?xml encoding="utf-8" ?>' . $html, LIBXML_NOWARNING | LIBXML_NOERROR);
|
||||
libxml_clear_errors();
|
||||
libxml_use_internal_errors($prev);
|
||||
|
||||
return $doc;
|
||||
}
|
||||
|
||||
private static function strip_bom(string $s): string {
|
||||
return str_starts_with($s, "\xEF\xBB\xBF") ? substr($s, 3) : $s;
|
||||
}
|
||||
|
||||
/** Effective base URL for a document: its <base href> if present, else $url. */
|
||||
public static function base_url(\DOMDocument $doc, string $url): string {
|
||||
$base = (new \DOMXPath($doc))->query('//base[@href]');
|
||||
if ($base && $base->length > 0) {
|
||||
$href = trim(($base->item(0) instanceof \DOMElement) ? $base->item(0)->getAttribute('href') : '');
|
||||
if ($href !== '') return self::resolve($href, $url);
|
||||
}
|
||||
return $url;
|
||||
}
|
||||
|
||||
/** Resolve a possibly-relative URL against a base. */
|
||||
public static function resolve(string $href, string $base): string {
|
||||
$href = trim($href);
|
||||
if ($href === '') return $base;
|
||||
if (preg_match('#^[a-z][a-z0-9+.-]*:#i', $href) || str_starts_with($href, '#')) return $href;
|
||||
|
||||
$b = parse_url($base);
|
||||
if (!$b || empty($b['scheme']) || empty($b['host'])) return $href;
|
||||
|
||||
$origin = $b['scheme'] . '://' . $b['host'] . (isset($b['port']) ? ':' . $b['port'] : '');
|
||||
|
||||
if (str_starts_with($href, '//')) return $b['scheme'] . ':' . $href;
|
||||
if (str_starts_with($href, '/')) return $origin . self::normalize_path($href);
|
||||
|
||||
$dir = isset($b['path']) ? preg_replace('#/[^/]*$#', '/', $b['path']) : '/';
|
||||
return $origin . self::normalize_path(($dir ?: '/') . $href);
|
||||
}
|
||||
|
||||
/** Collapse ./ and ../ segments. */
|
||||
private static function normalize_path(string $path): string {
|
||||
[$path, $tail] = array_pad(explode('?', $path, 2), 2, null);
|
||||
$out = [];
|
||||
|
||||
foreach (explode('/', $path) as $seg) {
|
||||
if ($seg === '.' || $seg === '') continue;
|
||||
if ($seg === '..') { array_pop($out); continue; }
|
||||
$out[] = $seg;
|
||||
}
|
||||
|
||||
$result = '/' . implode('/', $out);
|
||||
if (str_ends_with($path, '/') && !str_ends_with($result, '/')) $result .= '/';
|
||||
|
||||
return $tail !== null ? $result . '?' . $tail : $result;
|
||||
}
|
||||
|
||||
/** Rewrite every relative URL in the tree to an absolute one. */
|
||||
public static function absolutize(\DOMNode $ctx, string $base): void {
|
||||
$xpath = new \DOMXPath(self::owner($ctx));
|
||||
|
||||
foreach (['href', 'src', 'poster', 'data-src', 'longdesc'] as $attr) {
|
||||
foreach ($xpath->query(".//*[@$attr]", $ctx) ?: [] as $el) {
|
||||
if (!$el instanceof \DOMElement) continue;
|
||||
$v = $el->getAttribute($attr);
|
||||
if ($v !== '' && !str_starts_with($v, 'data:'))
|
||||
$el->setAttribute($attr, self::resolve($v, $base));
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($xpath->query('.//*[@srcset]', $ctx) ?: [] as $el) {
|
||||
if (!$el instanceof \DOMElement) continue;
|
||||
$el->setAttribute('srcset', self::absolutize_srcset($el->getAttribute('srcset'), $base));
|
||||
}
|
||||
}
|
||||
|
||||
private static function absolutize_srcset(string $srcset, string $base): string {
|
||||
$out = [];
|
||||
|
||||
foreach (explode(',', $srcset) as $part) {
|
||||
$part = trim($part);
|
||||
if ($part === '') continue;
|
||||
$bits = preg_split('/\s+/', $part, 2);
|
||||
$url = self::resolve($bits[0], $base);
|
||||
$out[] = isset($bits[1]) ? "$url {$bits[1]}" : $url;
|
||||
}
|
||||
|
||||
return implode(', ', $out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Promote lazy-loading placeholders to a real src.
|
||||
*
|
||||
* This is what makes webcomics work: the panel is routinely a 1x1 gif or a
|
||||
* data: URI until the site's JS swaps in data-src, so an extractor that only
|
||||
* reads @src produces an article with no picture in it.
|
||||
*/
|
||||
public static function unlazy(\DOMNode $ctx): void {
|
||||
$xpath = new \DOMXPath(self::owner($ctx));
|
||||
|
||||
foreach ($xpath->query('.//img', $ctx) ?: [] as $img) {
|
||||
if (!$img instanceof \DOMElement) continue;
|
||||
|
||||
$src = trim($img->getAttribute('src'));
|
||||
|
||||
if (!self::is_placeholder($src)) {
|
||||
// A real src, but a srcset may still offer a larger rendition.
|
||||
if ($src === '' && ($best = self::best_from_srcset($img->getAttribute('srcset'))))
|
||||
$img->setAttribute('src', $best);
|
||||
continue;
|
||||
}
|
||||
|
||||
$replacement = '';
|
||||
|
||||
foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url', 'data-full-src'] as $attr) {
|
||||
$v = trim($img->getAttribute($attr));
|
||||
if ($v !== '' && !self::is_placeholder($v)) { $replacement = $v; break; }
|
||||
}
|
||||
|
||||
if ($replacement === '')
|
||||
foreach (['data-srcset', 'srcset', 'data-lazy-srcset'] as $attr)
|
||||
if ($best = self::best_from_srcset($img->getAttribute($attr))) { $replacement = $best; break; }
|
||||
|
||||
if ($replacement !== '') $img->setAttribute('src', $replacement);
|
||||
|
||||
self::drop_lazy_attrs($img);
|
||||
}
|
||||
|
||||
// <picture><source srcset> with no usable <img src> underneath.
|
||||
foreach ($xpath->query('.//picture', $ctx) ?: [] as $pic) {
|
||||
if (!$pic instanceof \DOMElement) continue;
|
||||
|
||||
$imgs = (new \DOMXPath(self::owner($ctx)))->query('.//img', $pic);
|
||||
$img = ($imgs && $imgs->length) ? $imgs->item(0) : null;
|
||||
if (!$img instanceof \DOMElement || !self::is_placeholder(trim($img->getAttribute('src')))) continue;
|
||||
|
||||
foreach ((new \DOMXPath(self::owner($ctx)))->query('.//source[@srcset]', $pic) ?: [] as $source) {
|
||||
if (!$source instanceof \DOMElement) continue;
|
||||
if ($best = self::best_from_srcset($source->getAttribute('srcset'))) {
|
||||
$img->setAttribute('src', $best);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the lazy-loading attributes once their value has been promoted.
|
||||
*
|
||||
* Left in place they roughly double the stored article -- tapas.io signs every
|
||||
* panel URL, so each image carries its full token twice.
|
||||
*/
|
||||
private static function drop_lazy_attrs(\DOMElement $img): void {
|
||||
foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url',
|
||||
'data-full-src', 'data-srcset', 'data-lazy-srcset'] as $attr)
|
||||
if ($img->hasAttribute($attr)) $img->removeAttribute($attr);
|
||||
}
|
||||
|
||||
private static function is_placeholder(string $src): bool {
|
||||
if ($src === '') return true;
|
||||
if (str_starts_with($src, 'data:')) return true;
|
||||
|
||||
return (bool) preg_match('#(^|/)(blank|spacer|placeholder|transparent|lazy|1x1|pixel)[^/]*\.(gif|png|svg|webp)$#i', $src);
|
||||
}
|
||||
|
||||
/** Largest candidate in a srcset, by width or pixel density. */
|
||||
private static function best_from_srcset(string $srcset): ?string {
|
||||
$best = null;
|
||||
$best_score = -1.0;
|
||||
|
||||
foreach (explode(',', $srcset) as $part) {
|
||||
$part = trim($part);
|
||||
if ($part === '') continue;
|
||||
|
||||
$bits = preg_split('/\s+/', $part, 2);
|
||||
$url = $bits[0] ?? '';
|
||||
if ($url === '' || self::is_placeholder($url)) continue;
|
||||
|
||||
$score = 1.0;
|
||||
if (isset($bits[1]) && preg_match('/^([\d.]+)([wx])$/', trim($bits[1]), $m))
|
||||
$score = $m[2] === 'w' ? (float) $m[1] : (float) $m[1] * 1000;
|
||||
|
||||
if ($score > $best_score) { $best_score = $score; $best = $url; }
|
||||
}
|
||||
|
||||
return $best;
|
||||
}
|
||||
|
||||
/** Remove scripts, styles, inline event handlers and other non-content noise. */
|
||||
public static function sanitize(\DOMNode $ctx): void {
|
||||
$xpath = new \DOMXPath(self::owner($ctx));
|
||||
|
||||
foreach ($xpath->query('.//script | .//style | .//noscript | .//template | .//link | .//meta', $ctx) ?: [] as $el)
|
||||
$el->parentNode?->removeChild($el);
|
||||
|
||||
foreach ($xpath->query('.//@*', $ctx) ?: [] as $attr) {
|
||||
if (!$attr instanceof \DOMAttr) continue;
|
||||
|
||||
$name = strtolower($attr->name);
|
||||
$owner = $attr->ownerElement;
|
||||
if (!$owner) continue;
|
||||
|
||||
if (str_starts_with($name, 'on') || $name === 'style')
|
||||
$owner->removeAttribute($attr->name);
|
||||
|
||||
if (in_array($name, ['href', 'src', 'action'], true) && preg_match('/^\s*javascript:/i', $attr->value))
|
||||
$owner->removeAttribute($attr->name);
|
||||
}
|
||||
}
|
||||
|
||||
/** Serialize a node's children (its inner HTML). */
|
||||
public static function inner_html(\DOMNode $node): string {
|
||||
$doc = self::owner($node);
|
||||
$out = '';
|
||||
|
||||
foreach ($node->childNodes as $child)
|
||||
$out .= $doc->saveHTML($child);
|
||||
|
||||
return trim($out);
|
||||
}
|
||||
|
||||
public static function outer_html(\DOMNode $node): string {
|
||||
return trim((string) self::owner($node)->saveHTML($node));
|
||||
}
|
||||
|
||||
public static function text_length(\DOMNode $node): int {
|
||||
return mb_strlen(trim(preg_replace('/\s+/u', ' ', $node->textContent) ?? ''));
|
||||
}
|
||||
|
||||
private static function owner(\DOMNode $node): \DOMDocument {
|
||||
return $node instanceof \DOMDocument ? $node : ($node->ownerDocument ?? new \DOMDocument());
|
||||
}
|
||||
}
|
||||
68
lib/Rule.php
Normal file
68
lib/Rule.php
Normal file
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* A parsed FiveFilters site-config rule.
|
||||
*
|
||||
* Field names track the ftr-site-config directive names so a rule file and this
|
||||
* object read the same way. Directives that may repeat accumulate into arrays,
|
||||
* exactly as Full-Text RSS treats them.
|
||||
*/
|
||||
final class Rule {
|
||||
/** @var string[] */ public array $titles = [];
|
||||
/** @var string[] */ public array $bodies = [];
|
||||
/** @var string[] */ public array $authors = [];
|
||||
/** @var string[] */ public array $dates = [];
|
||||
/** @var string[] */ public array $strips = [];
|
||||
/** @var string[] */ public array $strip_id_or_class = [];
|
||||
/** @var string[] */ public array $strip_image_src = [];
|
||||
/** @var string[] */ public array $dissolve = [];
|
||||
/** @var string[] */ public array $single_page_links = [];
|
||||
/** @var string[] */ public array $next_page_links = [];
|
||||
/** @var string[] */ public array $find_strings = [];
|
||||
/** @var string[] */ public array $replace_strings = [];
|
||||
/** @var string[] */ public array $test_urls = [];
|
||||
/** @var array<string,string> */ public array $http_headers = [];
|
||||
/** @var array<string,string> map of xpath => wrapper spec e.g. "div.foo" */
|
||||
public array $wrap_in = [];
|
||||
|
||||
public bool $prune = true;
|
||||
public bool $tidy = true;
|
||||
public bool $autodetect_on_failure = true;
|
||||
|
||||
/**
|
||||
* Files this rule was assembled from, nearest-match first. Surfaced in the
|
||||
* UI so a stale rule is visible rather than silently falling through to
|
||||
* Readability -- the exact failure mode that hid a broken comic rule for
|
||||
* years.
|
||||
*
|
||||
* @var string[]
|
||||
*/
|
||||
public array $sources = [];
|
||||
|
||||
public function is_empty(): bool {
|
||||
return !$this->bodies && !$this->strips && !$this->strip_id_or_class
|
||||
&& !$this->strip_image_src && !$this->titles && !$this->dissolve;
|
||||
}
|
||||
|
||||
/** Merge $other UNDER $this: existing scalars win, list directives concatenate. */
|
||||
public function merge_under(self $other): self {
|
||||
$m = clone $this;
|
||||
|
||||
foreach (['titles', 'bodies', 'authors', 'dates', 'single_page_links', 'next_page_links'] as $f)
|
||||
if (!$m->$f) $m->$f = $other->$f;
|
||||
|
||||
foreach (['strips', 'strip_id_or_class', 'strip_image_src', 'dissolve', 'test_urls'] as $f)
|
||||
$m->$f = array_values(array_unique([...$m->$f, ...$other->$f]));
|
||||
|
||||
// find/replace are positional pairs -- appending keeps them aligned.
|
||||
$m->find_strings = [...$m->find_strings, ...$other->find_strings];
|
||||
$m->replace_strings = [...$m->replace_strings, ...$other->replace_strings];
|
||||
|
||||
$m->http_headers = $m->http_headers + $other->http_headers;
|
||||
$m->wrap_in = $m->wrap_in + $other->wrap_in;
|
||||
$m->sources = [...$m->sources, ...$other->sources];
|
||||
|
||||
return $m;
|
||||
}
|
||||
}
|
||||
148
lib/RuleSet.php
Normal file
148
lib/RuleSet.php
Normal file
@@ -0,0 +1,148 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* Loads ftr-site-config rule files and resolves the one that applies to a host.
|
||||
*
|
||||
* Two directories are searched, custom first: user rules written by the prefs UI
|
||||
* override the community set, and both use the same on-disk format so a rule
|
||||
* refined here can be sent upstream to fivefilters/ftr-site-config as-is.
|
||||
*/
|
||||
final class RuleSet {
|
||||
/** @var string[] directories searched in order (custom before standard) */
|
||||
private array $dirs;
|
||||
|
||||
/** @var array<string,?Rule> */
|
||||
private array $cache = [];
|
||||
|
||||
/** @param string[] $dirs */
|
||||
public function __construct(array $dirs) {
|
||||
$this->dirs = array_values(array_filter($dirs, 'is_dir'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the rule for a hostname, or null if nothing matches.
|
||||
*
|
||||
* Order follows Full-Text RSS: exact host, then the same host without a
|
||||
* leading "www.", then a wildcard file (".example.com.txt") for each parent
|
||||
* domain. global.txt is merged underneath whatever matched, so its shared
|
||||
* strip rules apply everywhere.
|
||||
*/
|
||||
public function find(string $host): ?Rule {
|
||||
$host = strtolower(trim($host));
|
||||
if ($host === '') return null;
|
||||
|
||||
if (array_key_exists($host, $this->cache))
|
||||
return $this->cache[$host];
|
||||
|
||||
$rule = null;
|
||||
|
||||
foreach ($this->candidates($host) as $name) {
|
||||
if ($found = $this->load($name)) { $rule = $found; break; }
|
||||
}
|
||||
|
||||
if ($global = $this->load('global')) {
|
||||
$rule = $rule ? $rule->merge_under($global) : $global;
|
||||
}
|
||||
|
||||
return $this->cache[$host] = $rule;
|
||||
}
|
||||
|
||||
/** Filenames to try, nearest match first. @return string[] */
|
||||
public function candidates(string $host): array {
|
||||
$out = [$host];
|
||||
|
||||
if (str_starts_with($host, 'www.'))
|
||||
$out[] = substr($host, 4);
|
||||
|
||||
// .example.com.txt applies to every subdomain of example.com.
|
||||
$parts = explode('.', $out[count($out) - 1]);
|
||||
while (count($parts) > 1) {
|
||||
$out[] = '.' . implode('.', $parts);
|
||||
array_shift($parts);
|
||||
}
|
||||
|
||||
return array_values(array_unique($out));
|
||||
}
|
||||
|
||||
/** Load one rule by bare name (no .txt), searching custom then standard. */
|
||||
public function load(string $name): ?Rule {
|
||||
// Rule names come from feed hostnames; keep them off the filesystem.
|
||||
if ($name === '' || str_contains($name, '/') || str_contains($name, "\0") || str_contains($name, '..'))
|
||||
return null;
|
||||
|
||||
foreach ($this->dirs as $dir) {
|
||||
$path = $dir . '/' . $name . '.txt';
|
||||
if (is_readable($path)) {
|
||||
$rule = self::parse((string) file_get_contents($path));
|
||||
$rule->sources[] = $path;
|
||||
return $rule;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Parse ftr-site-config text into a Rule. */
|
||||
public static function parse(string $text): Rule {
|
||||
$rule = new Rule();
|
||||
|
||||
foreach (preg_split('/\R/', $text) ?: [] as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
|
||||
$pos = strpos($line, ':');
|
||||
if ($pos === false) continue;
|
||||
|
||||
$key = strtolower(trim(substr($line, 0, $pos)));
|
||||
$val = trim(substr($line, $pos + 1));
|
||||
if ($val === '') continue;
|
||||
|
||||
// http_header(user-agent) / wrap_in(div.foo) carry an argument.
|
||||
$arg = null;
|
||||
if (preg_match('/^([a-z_]+)\((.*)\)$/', $key, $m)) {
|
||||
$key = $m[1];
|
||||
$arg = trim($m[2]);
|
||||
}
|
||||
|
||||
switch ($key) {
|
||||
case 'title': $rule->titles[] = $val; break;
|
||||
case 'body': $rule->bodies[] = $val; break;
|
||||
case 'author': $rule->authors[] = $val; break;
|
||||
case 'date': $rule->dates[] = $val; break;
|
||||
case 'strip': $rule->strips[] = $val; break;
|
||||
case 'strip_id_or_class': $rule->strip_id_or_class[] = $val; break;
|
||||
case 'strip_image_src': $rule->strip_image_src[] = $val; break;
|
||||
case 'dissolve': $rule->dissolve[] = $val; break;
|
||||
case 'single_page_link': $rule->single_page_links[] = $val; break;
|
||||
case 'next_page_link': $rule->next_page_links[] = $val; break;
|
||||
case 'find_string': $rule->find_strings[] = $val; break;
|
||||
case 'replace_string': $rule->replace_strings[] = $val; break;
|
||||
case 'test_url': $rule->test_urls[] = $val; break;
|
||||
|
||||
case 'prune': $rule->prune = self::truthy($val); break;
|
||||
case 'tidy': $rule->tidy = self::truthy($val); break;
|
||||
case 'autodetect_on_failure': $rule->autodetect_on_failure = self::truthy($val); break;
|
||||
|
||||
case 'http_header':
|
||||
if ($arg !== null && $arg !== '') $rule->http_headers[strtolower($arg)] = $val;
|
||||
break;
|
||||
|
||||
case 'wrap_in':
|
||||
if ($arg !== null && $arg !== '') $rule->wrap_in[$val] = $arg;
|
||||
break;
|
||||
|
||||
// Deliberately ignored in v1: if_page_contains, replace_string
|
||||
// variants with regex, convert_to_format, parser, src_lazy_load_attr
|
||||
// (we resolve lazy images unconditionally -- see Html::unlazy).
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
|
||||
return $rule;
|
||||
}
|
||||
|
||||
private static function truthy(string $v): bool {
|
||||
return in_array(strtolower($v), ['yes', 'true', '1', 'on'], true);
|
||||
}
|
||||
}
|
||||
48
lib/TtrssFetcher.php
Normal file
48
lib/TtrssFetcher.php
Normal file
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
namespace AfFulltext;
|
||||
|
||||
/**
|
||||
* The `direct` backend inside tt-rss.
|
||||
*
|
||||
* Deliberately goes through UrlHelper rather than cURL directly, so article
|
||||
* fetches keep tt-rss's own SSRF protection (UrlHelper::validate refuses private
|
||||
* and link-local addresses). Feeds are attacker-influenced input; a plugin that
|
||||
* fetches whatever a feed points at must not be the hole in that.
|
||||
*
|
||||
* FirecrawlFetcher is the deliberate exception -- it talks to one operator-configured
|
||||
* in-cluster endpoint, which UrlHelper would refuse precisely because it is private.
|
||||
*/
|
||||
final class TtrssFetcher implements Fetcher {
|
||||
public function __construct(private readonly int $timeout = 20) {}
|
||||
|
||||
public function name(): string { return 'direct'; }
|
||||
|
||||
public function fetch(string $url, array $headers = []): FetchResult {
|
||||
if (!class_exists('\UrlHelper'))
|
||||
return new FetchResult(error: 'UrlHelper unavailable (not running inside tt-rss)');
|
||||
|
||||
$options = [
|
||||
'url' => $url,
|
||||
'http_accept' => 'text/*',
|
||||
'type' => 'text/html',
|
||||
'timeout' => $this->timeout,
|
||||
];
|
||||
|
||||
// Site configs routinely set a user-agent to get the article rather than a
|
||||
// consent wall; UrlHelper takes it as a first-class option.
|
||||
if (isset($headers['user-agent'])) $options['useragent'] = $headers['user-agent'];
|
||||
|
||||
$body = \UrlHelper::fetch($options);
|
||||
|
||||
if (!is_string($body) || $body === '') {
|
||||
$error = \UrlHelper::$fetch_last_error ?: 'fetch returned nothing';
|
||||
return new FetchResult(error: $error, status: (int) \UrlHelper::$fetch_last_error_code);
|
||||
}
|
||||
|
||||
return new FetchResult(
|
||||
html: $body,
|
||||
effective_url: \UrlHelper::$fetch_effective_url ?: $url,
|
||||
status: 200,
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user