ttrss-plugin-af-fulltext/lib/Extractor.php

<?php
namespace AfFulltext;

use fivefilters\Readability\Configuration;
use fivefilters\Readability\Readability;

/**
 * Turns a URL into clean article HTML.
 *
 * Rule-driven where a site config exists, Readability where one does not. The
 * rule format is FiveFilters' own, so the ~2000 community rules in
 * fivefilters/ftr-site-config apply unmodified and anything written here can go
 * back upstream as a pull request.
 */
final class Extractor {
	private const MAX_PAGES = 5;

	public function __construct(
		private readonly RuleSet $rules,
		private readonly int $max_bytes = 4_000_000,
	) {}

	public function extract(string $url, Fetcher $fetcher): ExtractResult {
		$started = microtime(true);

		$result = new ExtractResult();
		$result->backend = $fetcher->name();
		$result->effective_url = $url;

		$host = strtolower((string) parse_url($url, PHP_URL_HOST));
		$rule = $this->rules->find($host);

		if ($rule) $result->rule_sources = $rule->sources;

		$fetched = $fetcher->fetch($url, $rule?->http_headers ?? []);

		if (!$fetched->ok()) {
			$result->errors[] = $fetched->error ?? 'fetch failed';
			$result->timing_ms = (int) round((microtime(true) - $started) * 1000);
			return $result;
		}

		if (strlen($fetched->html) > $this->max_bytes)
			$result->errors[] = sprintf('page is %dKB, truncating parse', strlen($fetched->html) / 1024);

		$result->effective_url = $fetched->effective_url ?: $url;

		$doc = Html::parse(substr($fetched->html, 0, $this->max_bytes));
		$base = Html::base_url($doc, $result->effective_url);

		// A print/single-page view, where one exists, is both cleaner and cheaper
		// to extract than following next_page_link through N requests.
		if ($rule && $rule->single_page_links) {
			$single = $this->first_url($doc, $rule->single_page_links, $base);

			if ($single && $single !== $result->effective_url) {
				$again = $fetcher->fetch($single, $rule->http_headers);

				if ($again->ok()) {
					$doc = Html::parse(substr($again->html, 0, $this->max_bytes));
					$result->effective_url = $again->effective_url ?: $single;
					$base = Html::base_url($doc, $result->effective_url);
				} else {
					$result->errors[] = 'single_page_link: ' . ($again->error ?? 'failed');
				}
			}
		}

		$html = $this->extract_from_doc($doc, $base, $rule, $result);

		// Multi-page articles: keep appending until the chain ends or we hit the cap.
		if ($rule && $rule->next_page_links) {
			$seen = [$result->effective_url => true];
			$current = $doc;
			$current_base = $base;

			while ($result->pages < self::MAX_PAGES) {
				$next = $this->first_url($current, $rule->next_page_links, $current_base);
				if (!$next || isset($seen[$next])) break;

				$seen[$next] = true;
				$page = $fetcher->fetch($next, $rule->http_headers);

				if (!$page->ok()) { $result->errors[] = 'next_page_link: ' . ($page->error ?? 'failed'); break; }

				$current = Html::parse(substr($page->html, 0, $this->max_bytes));
				$current_base = Html::base_url($current, $page->effective_url ?: $next);

				$more = $this->extract_from_doc($current, $current_base, $rule, new ExtractResult());
				if ($more === '') break;

				$html .= "\n" . $more;
				$result->pages++;
			}
		}

		if ($rule) {
			$html = $this->apply_replacements($html, $rule);
			$html = $this->apply_wrap_in($html, $rule);
		}

		$result->html = trim($html);
		$result->timing_ms = (int) round((microtime(true) - $started) * 1000);

		return $result;
	}

	/**
	 * Run the rule pipeline over one parsed document.
	 *
	 * Order matters: lazy images are resolved and URLs made absolute before
	 * anything is removed or selected, so every later step sees the same, final
	 * attribute values.
	 */
	private function extract_from_doc(\DOMDocument $doc, string $base, ?Rule $rule, ExtractResult $result): string {
		Html::unlazy($doc);
		Html::absolutize($doc, $base);

		$xpath = new \DOMXPath($doc);

		if ($rule) {
			$this->apply_removals($doc, $xpath, $rule);
			$this->apply_dissolve($xpath, $rule);

			$result->title ??= $this->first_string($xpath, $rule->titles);
			$result->author ??= $this->first_string($xpath, $rule->authors);
			$result->date ??= $this->first_string($xpath, $rule->dates);
		}

		if ($rule && $rule->bodies) {
			$nodes = $this->select_body($xpath, $rule->bodies);

			if ($nodes) {
				$result->rule_matched = true;

				$out = '';
				foreach ($nodes as $node) {
					Html::sanitize($node);
					$out .= Html::outer_html($node) . "\n";
				}

				return trim($out);
			}

			// A rule exists and selected nothing. Almost always a site redesign.
			$result->rule_stale = true;
			$result->errors[] = 'body rule matched no nodes (' . implode(', ', $rule->bodies) . ')';

			if (!$rule->autodetect_on_failure) return '';
		}

		$result->fell_back = true;

		return $this->readability($doc, $base, $result);
	}

	/** @return \DOMNode[] */
	private function select_body(\DOMXPath $xpath, array $expressions): array {
		$nodes = [];

		foreach ($expressions as $expr) {
			$found = @$xpath->query($expr);
			if ($found === false) continue;

			foreach ($found as $node) {
				if (Html::text_length($node) > 0 || $this->has_media($node)) $nodes[] = $node;
			}
		}

		return $nodes;
	}

	/**
	 * A node with no text is not necessarily empty -- on a webcomic the entire
	 * article is a single <img>, and discarding it for having no words is exactly
	 * the bug this plugin exists to fix.
	 */
	private const MEDIA_TAGS = ['img', 'video', 'audio', 'iframe', 'picture', 'source', 'svg', 'embed', 'object'];

	private function has_media(\DOMNode $node): bool {
		if (!$node instanceof \DOMElement && !$node instanceof \DOMDocument) return false;

		// The node may BE the media. `body: //img[@id='strip']` is a perfectly
		// ordinary rule for an image-only comic, and only looking at descendants
		// throws that selection away.
		if ($node instanceof \DOMElement && in_array(strtolower($node->tagName), self::MEDIA_TAGS, true))
			return true;

		$doc = $node instanceof \DOMDocument ? $node : $node->ownerDocument;
		if (!$doc) return false;

		$query = implode(' | ', array_map(fn(string $t) => ".//$t", self::MEDIA_TAGS));
		$found = @(new \DOMXPath($doc))->query($query, $node);

		return $found !== false && $found->length > 0;
	}

	private function apply_removals(\DOMDocument $doc, \DOMXPath $xpath, Rule $rule): void {
		foreach ($rule->strips as $expr) {
			$found = @$xpath->query($expr);
			if ($found === false) continue;
			foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
		}

		foreach ($rule->strip_id_or_class as $needle) {
			$q = $this->contains_lower('@id', $needle) . ' or ' . $this->contains_lower('@class', $needle);
			$found = @$xpath->query("//*[$q]");
			if ($found === false) continue;
			foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
		}

		foreach ($rule->strip_image_src as $needle) {
			$found = @$xpath->query('//img[' . $this->contains_lower('@src', $needle) . ']');
			if ($found === false) continue;
			foreach (iterator_to_array($found) as $node) $node->parentNode?->removeChild($node);
		}
	}

	/** Case-insensitive substring test, spelled out because XPath 1.0 has no lower-case(). */
	private function contains_lower(string $attr, string $needle): string {
		$upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
		$lower = 'abcdefghijklmnopqrstuvwxyz';
		$q = $this->xpath_literal(strtolower($needle));

		return "contains(translate($attr, '$upper', '$lower'), $q)";
	}

	/** Quote a string for XPath 1.0, which has no escape syntax. */
	private function xpath_literal(string $s): string {
		if (!str_contains($s, "'")) return "'$s'";
		if (!str_contains($s, '"')) return "\"$s\"";

		return 'concat(' . implode(", \"'\", ", array_map(fn($p) => "'$p'", explode("'", $s))) . ')';
	}

	/** Replace matched elements with their own children. */
	private function apply_dissolve(\DOMXPath $xpath, Rule $rule): void {
		foreach ($rule->dissolve as $expr) {
			$found = @$xpath->query($expr);
			if ($found === false) continue;

			foreach (iterator_to_array($found) as $node) {
				$parent = $node->parentNode;
				if (!$parent) continue;

				while ($node->firstChild) $parent->insertBefore($node->firstChild, $node);
				$parent->removeChild($node);
			}
		}
	}

	/**
	 * Readability fallback.
	 *
	 * The engine emits notices on malformed pages (null property reads on
	 * documents with no discernible body). Those must not reach output: inside
	 * tt-rss this runs mid-request and a stray warning corrupts the response. So
	 * diagnostics are captured into the result instead of being printed, and the
	 * previous handler is always restored.
	 */
	private function readability(\DOMDocument $doc, string $base, ExtractResult $result): string {
		$config = new Configuration();
		$config->setOriginalURL($base);
		$config->setFixRelativeURLs(true);
		$config->setParser('html5');
		// Class names carry meaning for downstream styling and for anyone writing
		// a rule off the extracted output.
		$config->setKeepClasses(true);

		$notices = [];

		set_error_handler(function (int $no, string $msg) use (&$notices): bool {
			$notices[] = $msg;
			return true;
		});

		try {
			$readability = new Readability($config);
			$readability->parse((string) $doc->saveHTML());

			$result->title ??= $readability->getTitle();
			$result->author ??= $readability->getAuthor();

			$content = (string) $readability->getContent();
		} catch (\Throwable $e) {
			$result->errors[] = 'readability: ' . $e->getMessage();
			$content = '';
		} finally {
			restore_error_handler();
		}

		if ($notices)
			$result->errors[] = sprintf('readability emitted %d notice(s): %s',
				count($notices), $notices[0]);

		return $content;
	}

	private function apply_replacements(string $html, Rule $rule): string {
		foreach ($rule->find_strings as $i => $find) {
			$replace = $rule->replace_strings[$i] ?? '';
			if ($find !== '') $html = str_replace($find, $replace, $html);
		}

		return $html;
	}

	private function apply_wrap_in(string $html, Rule $rule): string {
		// wrap_in targets nodes in the source document; applying it to the
		// already-serialized result would need a reparse for little gain, so v1
		// only honours a document-wide wrapper.
		foreach ($rule->wrap_in as $expr => $spec) {
			if ($expr !== '//*' && $expr !== '/') continue;

			[$tag, $class] = array_pad(explode('.', $spec, 2), 2, null);
			$tag = $tag ?: 'div';
			$attr = $class ? ' class="' . htmlspecialchars($class, ENT_QUOTES) . '"' : '';
			$html = "<$tag$attr>$html</$tag>";
		}

		return $html;
	}

	/** First non-empty string value produced by any of the expressions. */
	private function first_string(\DOMXPath $xpath, array $expressions): ?string {
		foreach ($expressions as $expr) {
			$value = @$xpath->evaluate($expr);

			if (is_string($value) && trim($value) !== '') return trim($value);

			if ($value instanceof \DOMNodeList && $value->length > 0) {
				$text = trim((string) $value->item(0)?->textContent);
				if ($text !== '') return $text;
			}
		}

		return null;
	}

	/** First absolute URL produced by any of the expressions. */
	private function first_url(\DOMDocument $doc, array $expressions, string $base): ?string {
		$xpath = new \DOMXPath($doc);

		foreach ($expressions as $expr) {
			$value = @$xpath->evaluate($expr);
			$raw = null;

			if (is_string($value) && trim($value) !== '') {
				$raw = trim($value);
			} elseif ($value instanceof \DOMNodeList && $value->length > 0) {
				$node = $value->item(0);
				$raw = $node instanceof \DOMElement ? ($node->getAttribute('href') ?: trim($node->textContent))
					: trim((string) $node?->nodeValue);
			}

			if ($raw) return Html::resolve($raw, $base);
		}

		return null;
	}
}