ttrss-plugin-af-fulltext/lib/Html.php

<?php
namespace AfFulltext;

use Masterminds\HTML5;

/** DOM helpers shared by the extractor. No tt-rss dependencies. */
final class Html {
	/**
	 * Parse a document. Masterminds' HTML5 parser handles modern markup that
	 * libxml mangles (unquoted attributes, <picture>, custom elements); libxml is
	 * the fallback because it copes with badly broken pages that make the HTML5
	 * parser give up.
	 */
	public static function parse(string $html): \DOMDocument {
		$html = self::strip_bom($html);

		try {
			$doc = (new HTML5(['disable_html_ns' => true]))->loadHTML($html);
			if ($doc->documentElement !== null) return $doc;
		} catch (\Throwable) {
			// fall through
		}

		$doc = new \DOMDocument();
		$prev = libxml_use_internal_errors(true);
		// Force UTF-8: libxml assumes ISO-8859-1 without a meta charset, which
		// silently mojibakes every non-ASCII article (most of these feeds are Polish).
		$doc->loadHTML('<?xml encoding="utf-8" ?>' . $html, LIBXML_NOWARNING | LIBXML_NOERROR);
		libxml_clear_errors();
		libxml_use_internal_errors($prev);

		return $doc;
	}

	private static function strip_bom(string $s): string {
		return str_starts_with($s, "\xEF\xBB\xBF") ? substr($s, 3) : $s;
	}

	/** Effective base URL for a document: its <base href> if present, else $url. */
	public static function base_url(\DOMDocument $doc, string $url): string {
		$base = (new \DOMXPath($doc))->query('//base[@href]');
		if ($base && $base->length > 0) {
			$href = trim(($base->item(0) instanceof \DOMElement) ? $base->item(0)->getAttribute('href') : '');
			if ($href !== '') return self::resolve($href, $url);
		}
		return $url;
	}

	/** Resolve a possibly-relative URL against a base. */
	public static function resolve(string $href, string $base): string {
		$href = trim($href);
		if ($href === '') return $base;
		if (preg_match('#^[a-z][a-z0-9+.-]*:#i', $href) || str_starts_with($href, '#')) return $href;

		$b = parse_url($base);
		if (!$b || empty($b['scheme']) || empty($b['host'])) return $href;

		$origin = $b['scheme'] . '://' . $b['host'] . (isset($b['port']) ? ':' . $b['port'] : '');

		if (str_starts_with($href, '//')) return $b['scheme'] . ':' . $href;
		if (str_starts_with($href, '/'))  return $origin . self::normalize_path($href);

		$dir = isset($b['path']) ? preg_replace('#/[^/]*$#', '/', $b['path']) : '/';
		return $origin . self::normalize_path(($dir ?: '/') . $href);
	}

	/** Collapse ./ and ../ segments. */
	private static function normalize_path(string $path): string {
		[$path, $tail] = array_pad(explode('?', $path, 2), 2, null);
		$out = [];

		foreach (explode('/', $path) as $seg) {
			if ($seg === '.' || $seg === '') continue;
			if ($seg === '..') { array_pop($out); continue; }
			$out[] = $seg;
		}

		$result = '/' . implode('/', $out);
		if (str_ends_with($path, '/') && !str_ends_with($result, '/')) $result .= '/';

		return $tail !== null ? $result . '?' . $tail : $result;
	}

	/** Rewrite every relative URL in the tree to an absolute one. */
	public static function absolutize(\DOMNode $ctx, string $base): void {
		$xpath = new \DOMXPath(self::owner($ctx));

		foreach (['href', 'src', 'poster', 'data-src', 'longdesc'] as $attr) {
			foreach ($xpath->query(".//*[@$attr]", $ctx) ?: [] as $el) {
				if (!$el instanceof \DOMElement) continue;
				$v = $el->getAttribute($attr);
				if ($v !== '' && !str_starts_with($v, 'data:'))
					$el->setAttribute($attr, self::resolve($v, $base));
			}
		}

		foreach ($xpath->query('.//*[@srcset]', $ctx) ?: [] as $el) {
			if (!$el instanceof \DOMElement) continue;
			$el->setAttribute('srcset', self::absolutize_srcset($el->getAttribute('srcset'), $base));
		}
	}

	private static function absolutize_srcset(string $srcset, string $base): string {
		$out = [];

		foreach (explode(',', $srcset) as $part) {
			$part = trim($part);
			if ($part === '') continue;
			$bits = preg_split('/\s+/', $part, 2);
			$url = self::resolve($bits[0], $base);
			$out[] = isset($bits[1]) ? "$url {$bits[1]}" : $url;
		}

		return implode(', ', $out);
	}

	/**
	 * Promote lazy-loading placeholders to a real src.
	 *
	 * This is what makes webcomics work: the panel is routinely a 1x1 gif or a
	 * data: URI until the site's JS swaps in data-src, so an extractor that only
	 * reads @src produces an article with no picture in it.
	 */
	public static function unlazy(\DOMNode $ctx): void {
		$xpath = new \DOMXPath(self::owner($ctx));

		foreach ($xpath->query('.//img', $ctx) ?: [] as $img) {
			if (!$img instanceof \DOMElement) continue;

			$src = trim($img->getAttribute('src'));

			if (!self::is_placeholder($src)) {
				// A real src, but a srcset may still offer a larger rendition.
				if ($src === '' && ($best = self::best_from_srcset($img->getAttribute('srcset'))))
					$img->setAttribute('src', $best);
				continue;
			}

			$replacement = '';

			foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url', 'data-full-src'] as $attr) {
				$v = trim($img->getAttribute($attr));
				if ($v !== '' && !self::is_placeholder($v)) { $replacement = $v; break; }
			}

			if ($replacement === '')
				foreach (['data-srcset', 'srcset', 'data-lazy-srcset'] as $attr)
					if ($best = self::best_from_srcset($img->getAttribute($attr))) { $replacement = $best; break; }

			if ($replacement !== '') $img->setAttribute('src', $replacement);

			self::drop_lazy_attrs($img);
		}

		// <picture><source srcset> with no usable <img src> underneath.
		foreach ($xpath->query('.//picture', $ctx) ?: [] as $pic) {
			if (!$pic instanceof \DOMElement) continue;

			$imgs = (new \DOMXPath(self::owner($ctx)))->query('.//img', $pic);
			$img = ($imgs && $imgs->length) ? $imgs->item(0) : null;
			if (!$img instanceof \DOMElement || !self::is_placeholder(trim($img->getAttribute('src')))) continue;

			foreach ((new \DOMXPath(self::owner($ctx)))->query('.//source[@srcset]', $pic) ?: [] as $source) {
				if (!$source instanceof \DOMElement) continue;
				if ($best = self::best_from_srcset($source->getAttribute('srcset'))) {
					$img->setAttribute('src', $best);
					break;
				}
			}
		}
	}

	/**
	 * Drop the lazy-loading attributes once their value has been promoted.
	 *
	 * Left in place they roughly double the stored article -- tapas.io signs every
	 * panel URL, so each image carries its full token twice.
	 */
	private static function drop_lazy_attrs(\DOMElement $img): void {
		foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url',
				'data-full-src', 'data-srcset', 'data-lazy-srcset'] as $attr)
			if ($img->hasAttribute($attr)) $img->removeAttribute($attr);
	}

	private static function is_placeholder(string $src): bool {
		if ($src === '') return true;
		if (str_starts_with($src, 'data:')) return true;

		return (bool) preg_match('#(^|/)(blank|spacer|placeholder|transparent|lazy|1x1|pixel)[^/]*\.(gif|png|svg|webp)$#i', $src);
	}

	/** Largest candidate in a srcset, by width or pixel density. */
	private static function best_from_srcset(string $srcset): ?string {
		$best = null;
		$best_score = -1.0;

		foreach (explode(',', $srcset) as $part) {
			$part = trim($part);
			if ($part === '') continue;

			$bits = preg_split('/\s+/', $part, 2);
			$url = $bits[0] ?? '';
			if ($url === '' || self::is_placeholder($url)) continue;

			$score = 1.0;
			if (isset($bits[1]) && preg_match('/^([\d.]+)([wx])$/', trim($bits[1]), $m))
				$score = $m[2] === 'w' ? (float) $m[1] : (float) $m[1] * 1000;

			if ($score > $best_score) { $best_score = $score; $best = $url; }
		}

		return $best;
	}

	/** Remove scripts, styles, inline event handlers and other non-content noise. */
	public static function sanitize(\DOMNode $ctx): void {
		$xpath = new \DOMXPath(self::owner($ctx));

		foreach ($xpath->query('.//script | .//style | .//noscript | .//template | .//link | .//meta', $ctx) ?: [] as $el)
			$el->parentNode?->removeChild($el);

		foreach ($xpath->query('.//@*', $ctx) ?: [] as $attr) {
			if (!$attr instanceof \DOMAttr) continue;

			$name = strtolower($attr->name);
			$owner = $attr->ownerElement;
			if (!$owner) continue;

			if (str_starts_with($name, 'on') || $name === 'style')
				$owner->removeAttribute($attr->name);

			if (in_array($name, ['href', 'src', 'action'], true) && preg_match('/^\s*javascript:/i', $attr->value))
				$owner->removeAttribute($attr->name);
		}
	}

	/** Serialize a node's children (its inner HTML). */
	public static function inner_html(\DOMNode $node): string {
		$doc = self::owner($node);
		$out = '';

		foreach ($node->childNodes as $child)
			$out .= $doc->saveHTML($child);

		return trim($out);
	}

	public static function outer_html(\DOMNode $node): string {
		return trim((string) self::owner($node)->saveHTML($node));
	}

	public static function text_length(\DOMNode $node): int {
		return mb_strlen(trim(preg_replace('/\s+/u', ' ', $node->textContent) ?? ''));
	}

	private static function owner(\DOMNode $node): \DOMDocument {
		return $node instanceof \DOMDocument ? $node : ($node->ownerDocument ?? new \DOMDocument());
	}
}