af_fulltext: rule-driven extraction with a pluggable renderer
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs go back to being real feed URLs and extraction happens inside tt-rss, using the same ftr-site-config rules the proxy used. The engine in lib/ has no tt-rss dependencies, so rules can be developed and audited from the command line; init.php is a thin adapter over it. Two findings from measuring the real subscription first, both of which shaped the design: - Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of chrome on a Cloudflare post), but it is an excellent renderer. So it is used for rawHtml only and the rule engine does the extraction. - A body rule that stops matching after a redesign falls through to Readability and still produces a plausible article, so the breakage is invisible. Every extraction now records which rule matched and whether it fell back; auditing the 34 live feeds surfaced five community rules that match nothing. Custom rules included for the sites that needed them, including three comics where the article is an image and text-scoring extractors return the wrong thing or nothing at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
This commit is contained in:
10
site_config/custom/andrzejrysuje.pl.txt
Normal file
10
site_config/custom/andrzejrysuje.pl.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
# Multi-panel strips are laid out as a Bootstrap carousel; taking carousel-inner
|
||||
# keeps every panel in order. The <figure> alone would give only the first one.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' carousel-inner ')]
|
||||
|
||||
strip_id_or_class: carousel-control
|
||||
strip_id_or_class: carousel-indicators
|
||||
|
||||
autodetect_on_failure: no
|
||||
|
||||
test_url: https://andrzejrysuje.pl/tradycja/
|
||||
12
site_config/custom/blog.cloudflare.com.txt
Normal file
12
site_config/custom/blog.cloudflare.com.txt
Normal file
@@ -0,0 +1,12 @@
|
||||
# The community rule still says //div[@class='post-content']. That class survives
|
||||
# the site's redesign but is now one token among many Tailwind classes, so an
|
||||
# exact-equality match never fires. article-content is the tighter target anyway.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' article-content ')]
|
||||
|
||||
title: //h1
|
||||
author: //a[contains(@href, '/author/')]
|
||||
|
||||
strip_id_or_class: post-email
|
||||
strip_id_or_class: image-lightbox
|
||||
|
||||
test_url: https://blog.cloudflare.com/bot-preference-sync/
|
||||
8
site_config/custom/dobreprogramy.pl.txt
Normal file
8
site_config/custom/dobreprogramy.pl.txt
Normal file
@@ -0,0 +1,8 @@
|
||||
# The old #phContent_divMetaBody id predates at least two rewrites of this site.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' article-body-grid ')]
|
||||
|
||||
strip_id_or_class: article-bottom
|
||||
strip_id_or_class: article-img-placeholder
|
||||
strip_id_or_class: adunit
|
||||
|
||||
test_url: https://www.dobreprogramy.pl/windows-11-26h2-coraz-blizej-mala-aktualizacja-z-szybka-instalacja,7321906159909056a
|
||||
4
site_config/custom/ianlewis.org.txt
Normal file
4
site_config/custom/ianlewis.org.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
# Now an h-entry microformat: "post e-content" rather than the old post-content.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' e-content ')]
|
||||
|
||||
test_url: https://www.ianlewis.org/en/2026-new-year-reflections
|
||||
12
site_config/custom/loadingartist.com.txt
Normal file
12
site_config/custom/loadingartist.com.txt
Normal file
@@ -0,0 +1,12 @@
|
||||
# The site was redesigned: the old rule (body: //div[@class='comic']) selects
|
||||
# nothing, so extraction silently fell through to Readability and dropped the
|
||||
# comic entirely -- the panel is the article, and Readability keeps text.
|
||||
body: //div[contains(@class, 'main-image-container')]
|
||||
body: //div[@class='post-content']
|
||||
|
||||
strip: //nav
|
||||
strip_id_or_class: pagination
|
||||
|
||||
autodetect_on_failure: no
|
||||
|
||||
test_url: https://loadingartist.com/comic/be-write-back/
|
||||
7
site_config/custom/oglaf.com.txt
Normal file
7
site_config/custom/oglaf.com.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
# Image-only page: the strip is the article. Readability returns nothing at all
|
||||
# here because there is no prose to score, which is why this needs a rule.
|
||||
body: //img[@id='strip']
|
||||
|
||||
autodetect_on_failure: no
|
||||
|
||||
test_url: https://www.oglaf.com/
|
||||
16
site_config/custom/securelist.com.txt
Normal file
16
site_config/custom/securelist.com.txt
Normal file
@@ -0,0 +1,16 @@
|
||||
# Redesigned onto a "c-" component prefix; the community rule's #primary wrapper
|
||||
# is long gone.
|
||||
body: //article[contains(concat(' ', normalize-space(@class), ' '), ' c-article ')]
|
||||
|
||||
# Sidebar widgets ("From the same authors", "In the same category", webinar
|
||||
# promos) and the collapsible table of contents all live inside c-article.
|
||||
strip_id_or_class: c-widget
|
||||
strip_id_or_class: js-sticky-widget
|
||||
strip_id_or_class: c-highlight
|
||||
strip_id_or_class: c-article__authors
|
||||
strip_id_or_class: c-card
|
||||
strip_id_or_class: c-share
|
||||
strip_id_or_class: comment
|
||||
strip_id_or_class: akismet
|
||||
|
||||
test_url: https://securelist.com/android-head-unit-malware/121106/
|
||||
10
site_config/custom/skeletonclaw.com.txt
Normal file
10
site_config/custom/skeletonclaw.com.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
# Tumblr: the strip sits in the post body; everything else on the page is site
|
||||
# furniture (logo, banner, Patreon button) that Readability happily keeps.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' post-content ')]
|
||||
|
||||
strip_id_or_class: post-notes
|
||||
strip_id_or_class: tumblr_controls
|
||||
|
||||
autodetect_on_failure: no
|
||||
|
||||
test_url: https://www.skeletonclaw.com/post/768403912930836480
|
||||
15
site_config/custom/tapas.io.txt
Normal file
15
site_config/custom/tapas.io.txt
Normal file
@@ -0,0 +1,15 @@
|
||||
# Panels are lazy-loaded: @src is a base64 placeholder gif and the real image is
|
||||
# in @data-src, which the extractor promotes. Without that the episode extracts
|
||||
# as a column of blank pixels.
|
||||
#
|
||||
# Note: tapas signs image URLs with an expiry, so a cached article's panels stop
|
||||
# resolving after a few weeks. Nothing to be done about that from this end.
|
||||
body: //div[contains(concat(' ', normalize-space(@class), ' '), ' js-episode-viewer ')]
|
||||
|
||||
strip_id_or_class: loading-indicator
|
||||
strip_id_or_class: episode-info
|
||||
strip_id_or_class: js-message-body
|
||||
|
||||
autodetect_on_failure: no
|
||||
|
||||
test_url: https://tapas.io/episode/2549532
|
||||
Reference in New Issue
Block a user