Files
ttrss-plugin-af-fulltext/lib/RuleSet.php
Michal 9bc854ec44 af_fulltext: rule-driven extraction with a pluggable renderer
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs
go back to being real feed URLs and extraction happens inside tt-rss, using the
same ftr-site-config rules the proxy used.

The engine in lib/ has no tt-rss dependencies, so rules can be developed and
audited from the command line; init.php is a thin adapter over it.

Two findings from measuring the real subscription first, both of which shaped
the design:

- Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of
  chrome on a Cloudflare post), but it is an excellent renderer. So it is used
  for rawHtml only and the rule engine does the extraction.
- A body rule that stops matching after a redesign falls through to Readability
  and still produces a plausible article, so the breakage is invisible. Every
  extraction now records which rule matched and whether it fell back; auditing
  the 34 live feeds surfaced five community rules that match nothing.

Custom rules included for the sites that needed them, including three comics
where the article is an image and text-scoring extractors return the wrong thing
or nothing at all.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
2026-08-25 00:16:16 +01:00

149 lines
4.8 KiB
PHP

<?php
namespace AfFulltext;
/**
* Loads ftr-site-config rule files and resolves the one that applies to a host.
*
* Two directories are searched, custom first: user rules written by the prefs UI
* override the community set, and both use the same on-disk format so a rule
* refined here can be sent upstream to fivefilters/ftr-site-config as-is.
*/
final class RuleSet {
/** @var string[] directories searched in order (custom before standard) */
private array $dirs;
/** @var array<string,?Rule> */
private array $cache = [];
/** @param string[] $dirs */
public function __construct(array $dirs) {
$this->dirs = array_values(array_filter($dirs, 'is_dir'));
}
/**
* Resolve the rule for a hostname, or null if nothing matches.
*
* Order follows Full-Text RSS: exact host, then the same host without a
* leading "www.", then a wildcard file (".example.com.txt") for each parent
* domain. global.txt is merged underneath whatever matched, so its shared
* strip rules apply everywhere.
*/
public function find(string $host): ?Rule {
$host = strtolower(trim($host));
if ($host === '') return null;
if (array_key_exists($host, $this->cache))
return $this->cache[$host];
$rule = null;
foreach ($this->candidates($host) as $name) {
if ($found = $this->load($name)) { $rule = $found; break; }
}
if ($global = $this->load('global')) {
$rule = $rule ? $rule->merge_under($global) : $global;
}
return $this->cache[$host] = $rule;
}
/** Filenames to try, nearest match first. @return string[] */
public function candidates(string $host): array {
$out = [$host];
if (str_starts_with($host, 'www.'))
$out[] = substr($host, 4);
// .example.com.txt applies to every subdomain of example.com.
$parts = explode('.', $out[count($out) - 1]);
while (count($parts) > 1) {
$out[] = '.' . implode('.', $parts);
array_shift($parts);
}
return array_values(array_unique($out));
}
/** Load one rule by bare name (no .txt), searching custom then standard. */
public function load(string $name): ?Rule {
// Rule names come from feed hostnames; keep them off the filesystem.
if ($name === '' || str_contains($name, '/') || str_contains($name, "\0") || str_contains($name, '..'))
return null;
foreach ($this->dirs as $dir) {
$path = $dir . '/' . $name . '.txt';
if (is_readable($path)) {
$rule = self::parse((string) file_get_contents($path));
$rule->sources[] = $path;
return $rule;
}
}
return null;
}
/** Parse ftr-site-config text into a Rule. */
public static function parse(string $text): Rule {
$rule = new Rule();
foreach (preg_split('/\R/', $text) ?: [] as $line) {
$line = trim($line);
if ($line === '' || $line[0] === '#') continue;
$pos = strpos($line, ':');
if ($pos === false) continue;
$key = strtolower(trim(substr($line, 0, $pos)));
$val = trim(substr($line, $pos + 1));
if ($val === '') continue;
// http_header(user-agent) / wrap_in(div.foo) carry an argument.
$arg = null;
if (preg_match('/^([a-z_]+)\((.*)\)$/', $key, $m)) {
$key = $m[1];
$arg = trim($m[2]);
}
switch ($key) {
case 'title': $rule->titles[] = $val; break;
case 'body': $rule->bodies[] = $val; break;
case 'author': $rule->authors[] = $val; break;
case 'date': $rule->dates[] = $val; break;
case 'strip': $rule->strips[] = $val; break;
case 'strip_id_or_class': $rule->strip_id_or_class[] = $val; break;
case 'strip_image_src': $rule->strip_image_src[] = $val; break;
case 'dissolve': $rule->dissolve[] = $val; break;
case 'single_page_link': $rule->single_page_links[] = $val; break;
case 'next_page_link': $rule->next_page_links[] = $val; break;
case 'find_string': $rule->find_strings[] = $val; break;
case 'replace_string': $rule->replace_strings[] = $val; break;
case 'test_url': $rule->test_urls[] = $val; break;
case 'prune': $rule->prune = self::truthy($val); break;
case 'tidy': $rule->tidy = self::truthy($val); break;
case 'autodetect_on_failure': $rule->autodetect_on_failure = self::truthy($val); break;
case 'http_header':
if ($arg !== null && $arg !== '') $rule->http_headers[strtolower($arg)] = $val;
break;
case 'wrap_in':
if ($arg !== null && $arg !== '') $rule->wrap_in[$val] = $arg;
break;
// Deliberately ignored in v1: if_page_contains, replace_string
// variants with regex, convert_to_format, parser, src_lazy_load_attr
// (we resolve lazy images unconditionally -- see Html::unlazy).
default: break;
}
}
return $rule;
}
private static function truthy(string $v): bool {
return in_array(strtolower($v), ['yes', 'true', '1', 'on'], true);
}
}