af_fulltext: rule-driven extraction with a pluggable renderer

Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs go back to being real feed URLs and extraction happens inside tt-rss, using the same ftr-site-config rules the proxy used. The engine in lib/ has no tt-rss dependencies, so rules can be developed and audited from the command line; init.php is a thin adapter over it. Two findings from measuring the real subscription first, both of which shaped the design: - Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of chrome on a Cloudflare post), but it is an excellent renderer. So it is used for rawHtml only and the rule engine does the extraction. - A body rule that stops matching after a redesign falls through to Readability and still produces a plausible article, so the breakage is invisible. Every extraction now records which rule matched and whether it fell back; auditing the 34 live feeds surfaced five community rules that match nothing. Custom rules included for the sites that needed them, including three comics where the article is an image and text-scoring extractors return the wrong thing or nothing at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
2026-08-25 00:16:16 +01:00
commit 9bc854ec44
146 changed files with 30221 additions and 0 deletions
--- a/bin/audit.php
+++ b/bin/audit.php
@@ -0,0 +1,117 @@
+#!/usr/bin/env php
+<?php
+/**
+ * Extraction health report across a list of feeds.
+ *
+ * Reads feed URLs (one per line) on stdin or from a file, pulls the newest item
+ * from each, extracts it, and reports what happened. The interesting column is
+ * `how`: a feed showing STALE has a rule that no longer matches, which is the
+ * failure that otherwise degrades silently.
+ *
+ *   php bin/audit.php feeds.txt [--backend=direct|firecrawl] [--items=1]
+ */
+require_once __DIR__ . '/../autoload.php';
+
+use AfFulltext\{CurlFetcher, Extractor, FirecrawlFetcher, RuleSet};
+
+$args = array_slice($argv, 1);
+$backend = 'direct';
+$items = 1;
+$file = 'php://stdin';
+
+foreach ($args as $arg) {
+	if (str_starts_with($arg, '--backend=')) $backend = substr($arg, 10);
+	elseif (str_starts_with($arg, '--items=')) $items = max(1, (int) substr($arg, 8));
+	elseif (!str_starts_with($arg, '--')) $file = $arg;
+}
+
+$base = dirname(__DIR__);
+$rules = new RuleSet([$base . '/site_config/custom', $base . '/site_config/standard']);
+$extractor = new Extractor($rules);
+
+$fetcher = $backend === 'firecrawl'
+	? new FirecrawlFetcher(getenv('FIRECRAWL_URL') ?: 'http://127.0.0.1:13002')
+	: new CurlFetcher();
+
+$feed_fetcher = new CurlFetcher();
+
+/** Newest item links in a feed, RSS or Atom. @return string[] */
+function item_links(string $xml, int $limit): array {
+	$prev = libxml_use_internal_errors(true);
+	$doc = simplexml_load_string($xml);
+	libxml_clear_errors();
+	libxml_use_internal_errors($prev);
+
+	if ($doc === false) return [];
+
+	$out = [];
+
+	// RSS 2.0 / RDF
+	foreach ($doc->xpath('//item') ?: [] as $item) {
+		$link = trim((string) $item->link);
+		if ($link !== '') $out[] = $link;
+		if (count($out) >= $limit) return $out;
+	}
+
+	// Atom
+	$doc->registerXPathNamespace('a', 'http://www.w3.org/2005/Atom');
+	foreach ($doc->xpath('//a:entry') ?: [] as $entry) {
+		$entry->registerXPathNamespace('a', 'http://www.w3.org/2005/Atom');
+		foreach ($entry->xpath('a:link[not(@rel) or @rel="alternate"]') ?: [] as $link) {
+			$href = trim((string) $link['href']);
+			if ($href !== '') { $out[] = $href; break; }
+		}
+		if (count($out) >= $limit) break;
+	}
+
+	return array_slice($out, 0, $limit);
+}
+
+$lines = array_filter(array_map('trim', file($file) ?: []), fn($l) => $l !== '' && !str_starts_with($l, '#'));
+
+printf("%-34s %-9s %-30s %7s %5s  %s\n", 'FEED HOST', 'BACKEND', 'HOW', 'BYTES', 'IMGS', 'NOTES');
+printf("%s\n", str_repeat('-', 130));
+
+$totals = ['ok' => 0, 'stale' => 0, 'readability' => 0, 'failed' => 0];
+
+foreach ($lines as $feed_url) {
+	$host = (string) parse_url($feed_url, PHP_URL_HOST);
+
+	$feed = $feed_fetcher->fetch($feed_url);
+	if (!$feed->ok()) {
+		printf("%-34s %-9s %-30s %7s %5s  %s\n", substr($host, 0, 34), '-', 'FEED FETCH FAILED', '-', '-', $feed->error);
+		$totals['failed']++;
+		continue;
+	}
+
+	$links = item_links($feed->html, $items);
+	if (!$links) {
+		printf("%-34s %-9s %-30s %7s %5s  %s\n", substr($host, 0, 34), '-', 'NO ITEMS', '-', '-', '');
+		$totals['failed']++;
+		continue;
+	}
+
+	foreach ($links as $link) {
+		$r = $extractor->extract($link, $fetcher);
+
+		$how = match (true) {
+			$r->rule_matched => 'rule: ' . basename($r->rule_sources[0] ?? '?'),
+			$r->rule_stale   => 'STALE: ' . basename($r->rule_sources[0] ?? '?'),
+			$r->ok()         => 'readability',
+			default          => 'FAILED',
+		};
+
+		if ($r->rule_matched) $totals['ok']++;
+		elseif ($r->rule_stale) $totals['stale']++;
+		elseif ($r->ok()) $totals['readability']++;
+		else $totals['failed']++;
+
+		printf("%-34s %-9s %-30s %7d %5d  %s\n",
+			substr((string) parse_url($link, PHP_URL_HOST), 0, 34),
+			$r->backend, substr($how, 0, 30), strlen($r->html),
+			substr_count($r->html, '<img'), implode('; ', array_slice($r->errors, 0, 1)));
+	}
+}
+
+printf("\n%d rule-matched, %d STALE rules, %d readability-only, %d failed\n",
+	$totals['ok'], $totals['stale'], $totals['readability'], $totals['failed']);
--- a/bin/extract.php
+++ b/bin/extract.php
@@ -0,0 +1,60 @@
+#!/usr/bin/env php
+<?php
+/**
+ * Standalone harness for the extraction engine.
+ *
+ * The whole point of keeping lib/ free of tt-rss dependencies: rules can be
+ * written and checked against real URLs without a running instance.
+ *
+ *   php bin/extract.php <url> [--backend=direct|firecrawl] [--rule=FILE] [--html] [--quiet]
+ */
+require_once __DIR__ . '/../autoload.php';
+
+use AfFulltext\{CurlFetcher, Extractor, FirecrawlFetcher, RuleSet};
+
+$args = array_slice($argv, 1);
+$opts = ['backend' => 'direct', 'rule' => null, 'html' => false, 'quiet' => false];
+$url = null;
+
+foreach ($args as $arg) {
+	if (str_starts_with($arg, '--backend=')) $opts['backend'] = substr($arg, 10);
+	elseif (str_starts_with($arg, '--rule=')) $opts['rule'] = substr($arg, 7);
+	elseif ($arg === '--html') $opts['html'] = true;
+	elseif ($arg === '--quiet') $opts['quiet'] = true;
+	elseif (!str_starts_with($arg, '--')) $url = $arg;
+}
+
+if (!$url) {
+	fwrite(STDERR, "usage: extract.php <url> [--backend=direct|firecrawl] [--rule=FILE] [--html] [--quiet]\n");
+	exit(2);
+}
+
+$base = dirname(__DIR__);
+$dirs = [$base . '/site_config/custom', $base . '/site_config/standard'];
+
+// --rule points at a single file to try, overriding whatever the site config says.
+if ($opts['rule']) {
+	$tmp = sys_get_temp_dir() . '/af_fulltext_rule_' . getmypid();
+	@mkdir($tmp, 0700, true);
+	$host = strtolower((string) parse_url($url, PHP_URL_HOST));
+	copy($opts['rule'], "$tmp/$host.txt");
+	array_unshift($dirs, $tmp);
+}
+
+$fetcher = $opts['backend'] === 'firecrawl'
+	? new FirecrawlFetcher(getenv('FIRECRAWL_URL') ?: 'http://127.0.0.1:13002')
+	: new CurlFetcher();
+
+$result = (new Extractor(new RuleSet($dirs)))->extract($url, $fetcher);
+
+if (!$opts['quiet']) {
+	fwrite(STDERR, $result->summary() . "\n");
+	foreach ($result->rule_sources as $s) fwrite(STDERR, "  rule: $s\n");
+	foreach ($result->errors as $e) fwrite(STDERR, "  error: $e\n");
+	if ($result->title) fwrite(STDERR, "  title: {$result->title}\n");
+}
+
+if ($opts['html']) echo $result->html, "\n";
+else echo trim(preg_replace('/\s+/u', ' ', strip_tags($result->html)) ?? ''), "\n";
+
+exit($result->ok() ? 0 : 1);
--- a/bin/php-podman.sh
+++ b/bin/php-podman.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Run the harness inside a pinned PHP image so results do not depend on whatever
+# PHP happens to be on the host (there is none on this workstation).
+exec podman run --rm --network=host -v "$(cd "$(dirname "$0")/.." && pwd)":/plugin:z -w /plugin \
+  docker.io/library/php:8.4-cli php "$@"