, custom elements); libxml is * the fallback because it copes with badly broken pages that make the HTML5 * parser give up. */ public static function parse(string $html): \DOMDocument { $html = self::strip_bom($html); try { $doc = (new HTML5(['disable_html_ns' => true]))->loadHTML($html); if ($doc->documentElement !== null) return $doc; } catch (\Throwable) { // fall through } $doc = new \DOMDocument(); $prev = libxml_use_internal_errors(true); // Force UTF-8: libxml assumes ISO-8859-1 without a meta charset, which // silently mojibakes every non-ASCII article (most of these feeds are Polish). $doc->loadHTML('' . $html, LIBXML_NOWARNING | LIBXML_NOERROR); libxml_clear_errors(); libxml_use_internal_errors($prev); return $doc; } private static function strip_bom(string $s): string { return str_starts_with($s, "\xEF\xBB\xBF") ? substr($s, 3) : $s; } /** Effective base URL for a document: its if present, else $url. */ public static function base_url(\DOMDocument $doc, string $url): string { $base = (new \DOMXPath($doc))->query('//base[@href]'); if ($base && $base->length > 0) { $href = trim(($base->item(0) instanceof \DOMElement) ? $base->item(0)->getAttribute('href') : ''); if ($href !== '') return self::resolve($href, $url); } return $url; } /** Resolve a possibly-relative URL against a base. */ public static function resolve(string $href, string $base): string { $href = trim($href); if ($href === '') return $base; if (preg_match('#^[a-z][a-z0-9+.-]*:#i', $href) || str_starts_with($href, '#')) return $href; $b = parse_url($base); if (!$b || empty($b['scheme']) || empty($b['host'])) return $href; $origin = $b['scheme'] . '://' . $b['host'] . (isset($b['port']) ? ':' . $b['port'] : ''); if (str_starts_with($href, '//')) return $b['scheme'] . ':' . $href; if (str_starts_with($href, '/')) return $origin . self::normalize_path($href); $dir = isset($b['path']) ? preg_replace('#/[^/]*$#', '/', $b['path']) : '/'; return $origin . self::normalize_path(($dir ?: '/') . $href); } /** Collapse ./ and ../ segments. */ private static function normalize_path(string $path): string { [$path, $tail] = array_pad(explode('?', $path, 2), 2, null); $out = []; foreach (explode('/', $path) as $seg) { if ($seg === '.' || $seg === '') continue; if ($seg === '..') { array_pop($out); continue; } $out[] = $seg; } $result = '/' . implode('/', $out); if (str_ends_with($path, '/') && !str_ends_with($result, '/')) $result .= '/'; return $tail !== null ? $result . '?' . $tail : $result; } /** Rewrite every relative URL in the tree to an absolute one. */ public static function absolutize(\DOMNode $ctx, string $base): void { $xpath = new \DOMXPath(self::owner($ctx)); foreach (['href', 'src', 'poster', 'data-src', 'longdesc'] as $attr) { foreach ($xpath->query(".//*[@$attr]", $ctx) ?: [] as $el) { if (!$el instanceof \DOMElement) continue; $v = $el->getAttribute($attr); if ($v !== '' && !str_starts_with($v, 'data:')) $el->setAttribute($attr, self::resolve($v, $base)); } } foreach ($xpath->query('.//*[@srcset]', $ctx) ?: [] as $el) { if (!$el instanceof \DOMElement) continue; $el->setAttribute('srcset', self::absolutize_srcset($el->getAttribute('srcset'), $base)); } } private static function absolutize_srcset(string $srcset, string $base): string { $out = []; foreach (explode(',', $srcset) as $part) { $part = trim($part); if ($part === '') continue; $bits = preg_split('/\s+/', $part, 2); $url = self::resolve($bits[0], $base); $out[] = isset($bits[1]) ? "$url {$bits[1]}" : $url; } return implode(', ', $out); } /** * Promote lazy-loading placeholders to a real src. * * This is what makes webcomics work: the panel is routinely a 1x1 gif or a * data: URI until the site's JS swaps in data-src, so an extractor that only * reads @src produces an article with no picture in it. */ public static function unlazy(\DOMNode $ctx): void { $xpath = new \DOMXPath(self::owner($ctx)); foreach ($xpath->query('.//img', $ctx) ?: [] as $img) { if (!$img instanceof \DOMElement) continue; $src = trim($img->getAttribute('src')); if (!self::is_placeholder($src)) { // A real src, but a srcset may still offer a larger rendition. if ($src === '' && ($best = self::best_from_srcset($img->getAttribute('srcset')))) $img->setAttribute('src', $best); continue; } $replacement = ''; foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url', 'data-full-src'] as $attr) { $v = trim($img->getAttribute($attr)); if ($v !== '' && !self::is_placeholder($v)) { $replacement = $v; break; } } if ($replacement === '') foreach (['data-srcset', 'srcset', 'data-lazy-srcset'] as $attr) if ($best = self::best_from_srcset($img->getAttribute($attr))) { $replacement = $best; break; } if ($replacement !== '') $img->setAttribute('src', $replacement); self::drop_lazy_attrs($img); } // with no usable underneath. foreach ($xpath->query('.//picture', $ctx) ?: [] as $pic) { if (!$pic instanceof \DOMElement) continue; $imgs = (new \DOMXPath(self::owner($ctx)))->query('.//img', $pic); $img = ($imgs && $imgs->length) ? $imgs->item(0) : null; if (!$img instanceof \DOMElement || !self::is_placeholder(trim($img->getAttribute('src')))) continue; foreach ((new \DOMXPath(self::owner($ctx)))->query('.//source[@srcset]', $pic) ?: [] as $source) { if (!$source instanceof \DOMElement) continue; if ($best = self::best_from_srcset($source->getAttribute('srcset'))) { $img->setAttribute('src', $best); break; } } } } /** * Drop the lazy-loading attributes once their value has been promoted. * * Left in place they roughly double the stored article -- tapas.io signs every * panel URL, so each image carries its full token twice. */ private static function drop_lazy_attrs(\DOMElement $img): void { foreach (['data-src', 'data-original', 'data-lazy-src', 'data-url', 'data-full-src', 'data-srcset', 'data-lazy-srcset'] as $attr) if ($img->hasAttribute($attr)) $img->removeAttribute($attr); } private static function is_placeholder(string $src): bool { if ($src === '') return true; if (str_starts_with($src, 'data:')) return true; return (bool) preg_match('#(^|/)(blank|spacer|placeholder|transparent|lazy|1x1|pixel)[^/]*\.(gif|png|svg|webp)$#i', $src); } /** Largest candidate in a srcset, by width or pixel density. */ private static function best_from_srcset(string $srcset): ?string { $best = null; $best_score = -1.0; foreach (explode(',', $srcset) as $part) { $part = trim($part); if ($part === '') continue; $bits = preg_split('/\s+/', $part, 2); $url = $bits[0] ?? ''; if ($url === '' || self::is_placeholder($url)) continue; $score = 1.0; if (isset($bits[1]) && preg_match('/^([\d.]+)([wx])$/', trim($bits[1]), $m)) $score = $m[2] === 'w' ? (float) $m[1] : (float) $m[1] * 1000; if ($score > $best_score) { $best_score = $score; $best = $url; } } return $best; } /** Remove scripts, styles, inline event handlers and other non-content noise. */ public static function sanitize(\DOMNode $ctx): void { $xpath = new \DOMXPath(self::owner($ctx)); foreach ($xpath->query('.//script | .//style | .//noscript | .//template | .//link | .//meta', $ctx) ?: [] as $el) $el->parentNode?->removeChild($el); foreach ($xpath->query('.//@*', $ctx) ?: [] as $attr) { if (!$attr instanceof \DOMAttr) continue; $name = strtolower($attr->name); $owner = $attr->ownerElement; if (!$owner) continue; if (str_starts_with($name, 'on') || $name === 'style') $owner->removeAttribute($attr->name); if (in_array($name, ['href', 'src', 'action'], true) && preg_match('/^\s*javascript:/i', $attr->value)) $owner->removeAttribute($attr->name); } } /** Serialize a node's children (its inner HTML). */ public static function inner_html(\DOMNode $node): string { $doc = self::owner($node); $out = ''; foreach ($node->childNodes as $child) $out .= $doc->saveHTML($child); return trim($out); } public static function outer_html(\DOMNode $node): string { return trim((string) self::owner($node)->saveHTML($node)); } public static function text_length(\DOMNode $node): int { return mb_strlen(trim(preg_replace('/\s+/u', ' ', $node->textContent) ?? '')); } private static function owner(\DOMNode $node): \DOMDocument { return $node instanceof \DOMDocument ? $node : ($node->ownerDocument ?? new \DOMDocument()); } }