ttrss-plugin-af-fulltext/bin/audit.php

#!/usr/bin/env php
<?php
/**
 * Extraction health report across a list of feeds.
 *
 * Reads feed URLs (one per line) on stdin or from a file, pulls the newest item
 * from each, extracts it, and reports what happened. The interesting column is
 * `how`: a feed showing STALE has a rule that no longer matches, which is the
 * failure that otherwise degrades silently.
 *
 *   php bin/audit.php feeds.txt [--backend=direct|firecrawl] [--items=1]
 */
require_once __DIR__ . '/../autoload.php';

use AfFulltext\{CurlFetcher, Extractor, FirecrawlFetcher, RuleSet};

$args = array_slice($argv, 1);
$backend = 'direct';
$items = 1;
$file = 'php://stdin';

foreach ($args as $arg) {
	if (str_starts_with($arg, '--backend=')) $backend = substr($arg, 10);
	elseif (str_starts_with($arg, '--items=')) $items = max(1, (int) substr($arg, 8));
	elseif (!str_starts_with($arg, '--')) $file = $arg;
}

$base = dirname(__DIR__);
$rules = new RuleSet([$base . '/site_config/custom', $base . '/site_config/standard']);
$extractor = new Extractor($rules);

$fetcher = $backend === 'firecrawl'
	? new FirecrawlFetcher(getenv('FIRECRAWL_URL') ?: 'http://127.0.0.1:13002')
	: new CurlFetcher();

$feed_fetcher = new CurlFetcher();

/** Newest item links in a feed, RSS or Atom. @return string[] */
function item_links(string $xml, int $limit): array {
	$prev = libxml_use_internal_errors(true);
	$doc = simplexml_load_string($xml);
	libxml_clear_errors();
	libxml_use_internal_errors($prev);

	if ($doc === false) return [];

	$out = [];

	// RSS 2.0 / RDF
	foreach ($doc->xpath('//item') ?: [] as $item) {
		$link = trim((string) $item->link);
		if ($link !== '') $out[] = $link;
		if (count($out) >= $limit) return $out;
	}

	// Atom
	$doc->registerXPathNamespace('a', 'http://www.w3.org/2005/Atom');
	foreach ($doc->xpath('//a:entry') ?: [] as $entry) {
		$entry->registerXPathNamespace('a', 'http://www.w3.org/2005/Atom');
		foreach ($entry->xpath('a:link[not(@rel) or @rel="alternate"]') ?: [] as $link) {
			$href = trim((string) $link['href']);
			if ($href !== '') { $out[] = $href; break; }
		}
		if (count($out) >= $limit) break;
	}

	return array_slice($out, 0, $limit);
}

$lines = array_filter(array_map('trim', file($file) ?: []), fn($l) => $l !== '' && !str_starts_with($l, '#'));

printf("%-34s %-9s %-30s %7s %5s  %s\n", 'FEED HOST', 'BACKEND', 'HOW', 'BYTES', 'IMGS', 'NOTES');
printf("%s\n", str_repeat('-', 130));

$totals = ['ok' => 0, 'stale' => 0, 'readability' => 0, 'failed' => 0];

foreach ($lines as $feed_url) {
	$host = (string) parse_url($feed_url, PHP_URL_HOST);

	$feed = $feed_fetcher->fetch($feed_url);
	if (!$feed->ok()) {
		printf("%-34s %-9s %-30s %7s %5s  %s\n", substr($host, 0, 34), '-', 'FEED FETCH FAILED', '-', '-', $feed->error);
		$totals['failed']++;
		continue;
	}

	$links = item_links($feed->html, $items);
	if (!$links) {
		printf("%-34s %-9s %-30s %7s %5s  %s\n", substr($host, 0, 34), '-', 'NO ITEMS', '-', '-', '');
		$totals['failed']++;
		continue;
	}

	foreach ($links as $link) {
		$r = $extractor->extract($link, $fetcher);

		$how = match (true) {
			$r->rule_matched => 'rule: ' . basename($r->rule_sources[0] ?? '?'),
			$r->rule_stale   => 'STALE: ' . basename($r->rule_sources[0] ?? '?'),
			$r->ok()         => 'readability',
			default          => 'FAILED',
		};

		if ($r->rule_matched) $totals['ok']++;
		elseif ($r->rule_stale) $totals['stale']++;
		elseif ($r->ok()) $totals['readability']++;
		else $totals['failed']++;

		printf("%-34s %-9s %-30s %7d %5d  %s\n",
			substr((string) parse_url($link, PHP_URL_HOST), 0, 34),
			$r->backend, substr($how, 0, 30), strlen($r->html),
			substr_count($r->html, '<img'), implode('; ', array_slice($r->errors, 0, 1)));
	}
}

printf("\n%d rule-matched, %d STALE rules, %d readability-only, %d failed\n",
	$totals['ok'], $totals['stale'], $totals['readability'], $totals['failed']);