ttrss-plugin-af-fulltext/init.php

<?php
/**
 * af_fulltext -- rule-driven full-text extraction for Tiny Tiny RSS.
 *
 * Replaces the usual arrangement of subscribing to a separate Full-Text RSS proxy
 * with extraction inside tt-rss, driven by the same fivefilters/ftr-site-config
 * rules that proxy used. Two fetch backends: tt-rss's own HTTP stack, or a
 * Firecrawl instance when a page only assembles itself under JavaScript.
 *
 * The extraction engine lives in lib/ and has no tt-rss dependencies, so rules can
 * be developed and audited from the command line (bin/extract.php, bin/audit.php).
 */
require_once __DIR__ . '/autoload.php';

use AfFulltext\{Extractor, FirecrawlFetcher, RuleSet, TtrssFetcher};
use AfFulltext\Fetcher;
use AfFulltext\ExtractResult;

class Af_Fulltext extends Plugin {
	private const BACKEND_DIRECT = 'direct';
	private const BACKEND_FIRECRAWL = 'firecrawl';

	/** Endpoint of a Firecrawl instance, e.g. http://firecrawl-api.firecrawl.svc.cluster.local:3002 */
	private const CONF_FIRECRAWL_URL = 'AF_FULLTEXT_FIRECRAWL_URL';
	private const CONF_FIRECRAWL_KEY = 'AF_FULLTEXT_FIRECRAWL_KEY';
	private const CONF_TIMEOUT = 'AF_FULLTEXT_TIMEOUT';
	/** Git repo the community rules are refreshed from. */
	private const CONF_RULES_REPO = 'AF_FULLTEXT_RULES_REPO';
	/** Hours between refreshes; 0 disables. */
	private const CONF_RULES_REFRESH_HOURS = 'AF_FULLTEXT_RULES_REFRESH_HOURS';

	/** @var PluginHost $host */
	private $host;

	function about() {
		return array(null,
			'Full-text extraction using ftr-site-config rules, with an optional Firecrawl renderer',
			'michal');
	}

	function flags() {
		return array('needs_curl' => true);
	}

	function api_version() {
		return 2;
	}

	function init($host) {
		$this->host = $host;

		Config::add(self::CONF_FIRECRAWL_URL, '', Config::T_STRING);
		Config::add(self::CONF_FIRECRAWL_KEY, '', Config::T_STRING);
		Config::add(self::CONF_TIMEOUT, '20', Config::T_INT);
		Config::add(self::CONF_RULES_REPO, 'https://github.com/fivefilters/ftr-site-config.git', Config::T_STRING);
		Config::add(self::CONF_RULES_REFRESH_HOURS, '24', Config::T_INT);

		$host->add_hook($host::HOOK_ARTICLE_FILTER, $this);
		$host->add_hook($host::HOOK_PREFS_TAB, $this);
		$host->add_hook($host::HOOK_PREFS_EDIT_FEED, $this);
		$host->add_hook($host::HOOK_PREFS_SAVE_FEED, $this);
		$host->add_hook($host::HOOK_ARTICLE_BUTTON, $this);
		$host->add_hook($host::HOOK_HOUSE_KEEPING, $this);

		// Installed unconditionally: init() runs before plugin storage is loaded,
		// so the enable flag cannot be consulted here.
		$host->add_hook($host::HOOK_GET_FULL_TEXT, $this);

		$host->add_filter_action($this, 'action_inline', __('Extract full text'));
		$host->add_filter_action($this, 'action_inline_append', __('Append full text'));
	}

	function get_js() {
		return file_get_contents(__DIR__ . '/init.js');
	}

	// ---------------------------------------------------------------- extraction

	private function rules_dirs(): array {
		return [__DIR__ . '/site_config/custom', __DIR__ . '/site_config/standard'];
	}

	private function extractor(): Extractor {
		return new Extractor(new RuleSet($this->rules_dirs()));
	}

	private function fetcher(string $backend): Fetcher {
		$timeout = (int) Config::get(self::CONF_TIMEOUT);

		if ($backend === self::BACKEND_FIRECRAWL) {
			$endpoint = (string) Config::get(self::CONF_FIRECRAWL_URL);

			if ($endpoint !== '')
				return new FirecrawlFetcher($endpoint, max($timeout, 60),
					((string) Config::get(self::CONF_FIRECRAWL_KEY)) ?: null);

			// Configured per feed but unavailable globally: fall back rather than
			// silently producing nothing.
			user_error('af_fulltext: firecrawl requested but ' . self::CONF_FIRECRAWL_URL . ' is unset', E_USER_WARNING);
		}

		return new TtrssFetcher($timeout);
	}

	/** Backend selected for a feed, defaulting to direct. */
	private function backend_for(int $feed_id): string {
		$map = $this->host->get_array($this, 'backend_feeds');

		return ($map[$feed_id] ?? self::BACKEND_DIRECT) === self::BACKEND_FIRECRAWL
			? self::BACKEND_FIRECRAWL
			: self::BACKEND_DIRECT;
	}

	private function extract(string $url, string $backend): ExtractResult {
		return $this->extractor()->extract($url, $this->fetcher($backend));
	}

	/**
	 * Record what happened for a feed.
	 *
	 * A `body:` rule that stops matching after a site redesign still yields a
	 * plausible-looking article via the Readability fallback, so the degradation is
	 * invisible in the reader. Keeping the last outcome per feed is what lets the
	 * settings pane say so out loud.
	 *
	 * @return void
	 */
	private function record_health(int $feed_id, ExtractResult $result) {
		$health = $this->host->get_array($this, 'health');

		$health[$feed_id] = [
			'when' => time(),
			'summary' => $result->summary(),
			'stale' => $result->rule_stale,
			'fell_back' => $result->fell_back,
			'bytes' => strlen($result->html),
			'error' => $result->errors[0] ?? null,
		];

		$this->host->set($this, 'health', $health);
	}

	/**
	 * @param array<string,mixed> $article
	 * @return array<string,mixed>
	 */
	function process_article(array $article, bool $append_mode, ?string $backend = null): array {
		$link = $article['link'] ?? '';
		if (!$link) return $article;

		$feed_id = (int) ($article['feed']['id'] ?? 0);
		$backend ??= $this->backend_for($feed_id);

		$result = $this->extract($link, $backend);

		if ($feed_id) $this->record_health($feed_id, $result);

		// Only replace the feed's own summary if we actually got something.
		if (!$this->has_content($result->html)) return $article;

		$content = $result->html . $this->provenance_comment($result);

		if ($append_mode) $article['content'] .= '<hr/>' . $content;
		else $article['content'] = $content;

		return $article;
	}

	/**
	 * An HTML comment naming the rule that produced the article.
	 *
	 * Invisible while reading, and the first thing worth looking at when an
	 * article comes out wrong.
	 */
	private function provenance_comment(ExtractResult $result): string {
		return "\n<!-- af_fulltext: " . htmlspecialchars($result->summary(), ENT_QUOTES) . " -->";
	}

	function hook_article_filter($article) {
		$enabled = $this->host->get_array($this, 'enabled_feeds');
		$append = $this->host->get_array($this, 'append_feeds');

		$feed_id = $article['feed']['id'] ?? null;

		if ($feed_id === null || !in_array($feed_id, $enabled)) return $article;

		return $this->process_article($article, in_array($feed_id, $append));
	}

	function hook_article_filter_action($article, $action) {
		return match ($action) {
			'action_inline' => $this->process_article($article, false),
			'action_inline_append' => $this->process_article($article, true),
			default => $article,
		};
	}

	function hook_get_full_text($link) {
		if (!$this->host->get($this, 'enable_share_anything')) return false;

		$result = $this->extract($link, self::BACKEND_DIRECT);

		return $this->has_content($result->html) ? $result->html : false;
	}

	/**
	 * Does this survive sanitising as something worth showing?
	 *
	 * Sanitizer::sanitize returns false on failure, so its result cannot be passed
	 * straight to strip_tags. An image-only article has no text at all and must
	 * still count -- for a webcomic the picture IS the content.
	 */
	private function has_content(string $html): bool {
		if ($html === '') return false;

		$clean = Sanitizer::sanitize($html);
		if (!is_string($clean)) return false;

		if (trim(strip_tags($clean)) !== '') return true;

		return (bool) preg_match('/<(img|video|audio|iframe|picture|source|svg|embed)\b/i', $clean);
	}

	// ------------------------------------------------------------- rule refresh

	/**
	 * Keep the community rules current.
	 *
	 * Runs from the update daemon rather than a separate scheduled job, so the
	 * rules travel with the plugin instead of being someone else's cron entry to
	 * remember. Refreshing is a fast-forward pull only: a conflict means someone
	 * edited the checkout by hand, and overwriting it silently would be worse than
	 * leaving it stale and saying so.
	 *
	 * @return void
	 */
	function hook_house_keeping() {
		$hours = (int) Config::get(self::CONF_RULES_REFRESH_HOURS);
		if ($hours <= 0) return;

		$last = (int) $this->host->get($this, 'rules_refreshed_at');
		if ($last && time() - $last < $hours * 3600) return;

		$dir = __DIR__ . '/site_config/standard';
		$repo = (string) Config::get(self::CONF_RULES_REPO);

		if (!is_dir("$dir/.git")) {
			Debug::log("af_fulltext: $dir is not a git checkout, skipping rule refresh", Debug::LOG_VERBOSE);
			return;
		}

		$cmd = sprintf('cd %s && git fetch --quiet --depth 1 origin 2>&1 && git reset --quiet --hard FETCH_HEAD 2>&1',
			escapeshellarg($dir));

		$output = [];
		$rc = 0;
		exec($cmd, $output, $rc);

		// Stamp the attempt either way, so a persistently unreachable remote does
		// not retry on every single housekeeping pass.
		$this->host->set($this, 'rules_refreshed_at', time());

		if ($rc !== 0) {
			$this->host->set($this, 'rules_refresh_error', implode(' ', array_slice($output, 0, 3)));
			Debug::log('af_fulltext: rule refresh failed: ' . implode(' ', $output), Debug::LOG_VERBOSE);
			return;
		}

		$this->host->set($this, 'rules_refresh_error', '');
		Debug::log("af_fulltext: refreshed site rules from $repo", Debug::LOG_VERBOSE);
	}

	// --------------------------------------------------------------------- UI

	function hook_article_button($line) {
		return "<i class='material-icons' onclick=\"Plugins.Af_Fulltext.embed(" . $line['id'] . ")\"
			style='cursor : pointer' title=\"" . __('Extract full text') . "\">article</i>";
	}

	function embed(): void {
		$article_id = (int) $_REQUEST['id'];

		$sth = $this->pdo->prepare('SELECT link, feed_id FROM ttrss_entries e, ttrss_user_entries ue
			WHERE e.id = ? AND ue.ref_id = e.id AND ue.owner_uid = ?');
		$sth->execute([$article_id, $_SESSION['uid']]);

		$ret = [];

		if ($row = $sth->fetch()) {
			$result = $this->extract($row['link'], $this->backend_for((int) $row['feed_id']));

			$ret['content'] = (string) Sanitizer::sanitize($result->html);
			$ret['summary'] = $result->summary();
			$ret['errors'] = $result->errors;
		}

		print json_encode($ret);
	}

	/** @return void */
	function save() {
		$this->host->set($this, 'enable_share_anything',
			checkbox_to_sql_bool($_POST['enable_share_anything'] ?? ''));

		echo __('Data saved.');
	}

	function hook_prefs_edit_feed($feed_id) {
		$enabled = $this->host->get_array($this, 'enabled_feeds');
		$append = $this->host->get_array($this, 'append_feeds');
		$backend = $this->backend_for((int) $feed_id);
		?>

		<header><?= __('Full-text extraction') ?></header>
		<section>
			<fieldset>
				<label class='checkbox'>
					<?= \Controls\checkbox_tag('af_fulltext_enabled', in_array($feed_id, $enabled)) ?>
					<?= __('Extract full article content') ?>
				</label>
			</fieldset>
			<fieldset>
				<label class='checkbox'>
					<?= \Controls\checkbox_tag('af_fulltext_append', in_array($feed_id, $append)) ?>
					<?= __('Append to the summary instead of replacing it') ?>
				</label>
			</fieldset>
			<fieldset>
				<label><?= __('Fetch using') ?></label>
				<?= \Controls\select_hash('af_fulltext_backend', $backend, [
					self::BACKEND_DIRECT => __('Direct (fast)'),
					self::BACKEND_FIRECRAWL => __('Firecrawl (renders JavaScript)'),
				]) ?>
			</fieldset>
		</section>
		<?php
	}

	function hook_prefs_save_feed($feed_id) {
		$enabled = $this->toggle($this->host->get_array($this, 'enabled_feeds'), $feed_id,
			(bool) checkbox_to_sql_bool($_POST['af_fulltext_enabled'] ?? ''));

		$append = $this->toggle($this->host->get_array($this, 'append_feeds'), $feed_id,
			(bool) checkbox_to_sql_bool($_POST['af_fulltext_append'] ?? ''));

		$backends = $this->host->get_array($this, 'backend_feeds');
		$chosen = $_POST['af_fulltext_backend'] ?? self::BACKEND_DIRECT;

		if ($chosen === self::BACKEND_FIRECRAWL) $backends[$feed_id] = self::BACKEND_FIRECRAWL;
		else unset($backends[$feed_id]);

		$this->host->set($this, 'enabled_feeds', $enabled);
		$this->host->set($this, 'append_feeds', $append);
		$this->host->set($this, 'backend_feeds', $backends);
	}

	/**
	 * @param array<int> $list
	 * @return array<int>
	 */
	private function toggle(array $list, $feed_id, bool $on): array {
		$key = array_search($feed_id, $list);

		if ($on && $key === false) $list[] = $feed_id;
		elseif (!$on && $key !== false) unset($list[$key]);

		return array_values($list);
	}

	function hook_prefs_tab($args) {
		if ($args != 'prefFeeds') return;

		$enable_share_anything = sql_bool_to_bool($this->host->get($this, 'enable_share_anything'));

		$enabled = $this->filter_unknown_feeds($this->host->get_array($this, 'enabled_feeds'));
		$this->host->set($this, 'enabled_feeds', $enabled);

		$append = $this->host->get_array($this, 'append_feeds');
		$health = $this->host->get_array($this, 'health');
		$refreshed = (int) $this->host->get($this, 'rules_refreshed_at');
		$refresh_error = (string) $this->host->get($this, 'rules_refresh_error');

		$rule_count = count(glob(__DIR__ . '/site_config/standard/*.txt') ?: [])
			+ count(glob(__DIR__ . '/site_config/custom/*.txt') ?: []);

		$degraded = array_filter($health, fn($h) => !empty($h['stale']));
		?>
		<div dojoType='dijit.layout.AccordionPane'
			title="<i class='material-icons'>article</i> <?= __('Full-text extraction (af_fulltext)') ?>">

			<?= format_notice('Enable per feed in the feed editor. ' . $rule_count . ' site rules loaded'
				. ($refreshed ? ', refreshed ' . date('Y-m-d H:i', $refreshed) : ', never refreshed') . '.') ?>

			<?php if ($refresh_error) { ?>
				<?= format_warning('Rule refresh failed: ' . htmlspecialchars($refresh_error)) ?>
			<?php } ?>

			<?php if ($degraded) { ?>
				<?= format_warning(sprintf(
					'%d feed(s) have a site rule that no longer matches anything and are falling back to Readability. '
					. 'That usually means the site was redesigned and the rule needs updating.',
					count($degraded))) ?>
			<?php } ?>

			<form dojoType='dijit.form.Form'>
				<?= \Controls\pluginhandler_tags($this, 'save') ?>

				<script type="dojo/method" event="onSubmit" args="evt">
					evt.preventDefault();
					if (this.validate()) {
						Notify.progress('Saving data...', true);
						xhr.post("backend.php", this.getValues(), (reply) => {
							Notify.info(reply);
						})
					}
				</script>

				<fieldset>
					<label class='checkbox'>
						<?= \Controls\checkbox_tag('enable_share_anything', $enable_share_anything) ?>
						<?= __('Provide full-text services to core code (bookmarklets) and other plugins') ?>
					</label>
				</fieldset>

				<hr/>
				<?= \Controls\submit_tag(__('Save')) ?>
			</form>

			<?php if (count($enabled) > 0) { ?>
				<hr/>
				<h3><?= __('Currently enabled for (click to edit):') ?></h3>

				<ul class='panel panel-scrollable list list-unstyled'>
					<?php foreach ($enabled as $f) {
						$h = $health[$f] ?? null;
						?>
						<li>
							<?php if (Feeds::_has_icon($f)) { ?>
								<img src='<?= Feeds::_get_icon_url($f) ?>' style="max-height: 20px" />
							<?php } else { ?> <i class='material-icons'>rss_feed</i> <?php } ?>

							<a href='#' onclick="CommonDialogs.editFeed(<?= $f ?>)">
								<?= Feeds::_get_title($f, $this->host->get_owner_uid()) ?>
							</a>

							<?= in_array($f, $append) ? ' ' . __('(append)') : '' ?>

							<?php if ($h) { ?>
								<small style="opacity: .7"><?= htmlspecialchars($h['summary'] ?? '') ?></small>
							<?php } ?>
						</li>
					<?php } ?>
				</ul>
			<?php } ?>
		</div>
		<?php
	}

	/**
	 * @param array<int> $feeds
	 * @return array<int>
	 */
	private function filter_unknown_feeds(array $feeds): array {
		$out = [];

		foreach ($feeds as $feed) {
			$sth = $this->pdo->prepare('SELECT id FROM ttrss_feeds WHERE id = ? AND owner_uid = ?');
			$sth->execute([$feed, $_SESSION['uid']]);

			if ($sth->fetch()) $out[] = $feed;
		}

		return $out;
	}
}