9bc854ec44
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs go back to being real feed URLs and extraction happens inside tt-rss, using the same ftr-site-config rules the proxy used. The engine in lib/ has no tt-rss dependencies, so rules can be developed and audited from the command line; init.php is a thin adapter over it. Two findings from measuring the real subscription first, both of which shaped the design: - Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of chrome on a Cloudflare post), but it is an excellent renderer. So it is used for rawHtml only and the rule engine does the extraction. - A body rule that stops matching after a redesign falls through to Readability and still produces a plausible article, so the breakage is invisible. Every extraction now records which rule matched and whether it fell back; auditing the 34 live feeds surfaced five community rules that match nothing. Custom rules included for the sites that needed them, including three comics where the article is an image and text-scoring extractors return the wrong thing or nothing at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
119 lines
3.3 KiB
PHP
119 lines
3.3 KiB
PHP
<?php
|
|
/**
|
|
* @file
|
|
* Test the Tree Builder's special-case rules.
|
|
*/
|
|
|
|
namespace Masterminds\HTML5\Tests\Parser;
|
|
|
|
use Masterminds\HTML5\Parser\TreeBuildingRules;
|
|
use Masterminds\HTML5\Parser\Tokenizer;
|
|
use Masterminds\HTML5\Parser\Scanner;
|
|
use Masterminds\HTML5\Parser\DOMTreeBuilder;
|
|
|
|
/**
|
|
* These tests are functional, not necessarily unit tests.
|
|
*/
|
|
class TreeBuildingRulesTest extends \Masterminds\HTML5\Tests\TestCase
|
|
{
|
|
const HTML_STUB = '<!DOCTYPE html><html><head><title>test</title></head><body>%s</body></html>';
|
|
|
|
/**
|
|
* Convenience function for parsing.
|
|
*/
|
|
protected function parse($string)
|
|
{
|
|
$treeBuilder = new DOMTreeBuilder();
|
|
$scanner = new Scanner($string);
|
|
$parser = new Tokenizer($scanner, $treeBuilder);
|
|
|
|
$parser->parse();
|
|
|
|
return $treeBuilder->document();
|
|
}
|
|
|
|
/**
|
|
* Convenience function for parsing fragments.
|
|
*/
|
|
protected function parseFragment($string)
|
|
{
|
|
$events = new DOMTreeBuilder(true);
|
|
$scanner = new Scanner($string);
|
|
$parser = new Tokenizer($scanner, $events);
|
|
|
|
$parser->parse();
|
|
|
|
return $events->fragment();
|
|
}
|
|
|
|
public function testTDFragment()
|
|
{
|
|
$frag = $this->parseFragment('<td>This is a test of the HTML5 parser</td>');
|
|
|
|
$td = $frag->childNodes->item(0);
|
|
|
|
$this->assertEquals(1, $frag->childNodes->length);
|
|
$this->assertEquals('td', $td->tagName);
|
|
$this->assertEquals('This is a test of the HTML5 parser', $td->nodeValue);
|
|
}
|
|
|
|
public function testHasRules()
|
|
{
|
|
$doc = new \DOMDocument('1.0');
|
|
$engine = new TreeBuildingRules($doc);
|
|
|
|
$this->assertTrue($engine->hasRules('li'));
|
|
$this->assertFalse($engine->hasRules('imaginary'));
|
|
}
|
|
|
|
public function testHandleLI()
|
|
{
|
|
$html = sprintf(self::HTML_STUB, '<ul id="a"><li>test<li>test2</ul><a></a>');
|
|
$doc = $this->parse($html);
|
|
|
|
$list = $doc->getElementById('a');
|
|
|
|
$this->assertEquals(2, $list->childNodes->length);
|
|
foreach ($list->childNodes as $ele) {
|
|
$this->assertEquals('li', $ele->tagName);
|
|
}
|
|
}
|
|
|
|
public function testHandleDT()
|
|
{
|
|
$html = sprintf(self::HTML_STUB, '<dl id="a"><dt>Hello<dd>Hi</dl><a></a>');
|
|
$doc = $this->parse($html);
|
|
|
|
$list = $doc->getElementById('a');
|
|
|
|
$this->assertEquals(2, $list->childNodes->length);
|
|
$this->assertEquals('dt', $list->firstChild->tagName);
|
|
$this->assertEquals('dd', $list->lastChild->tagName);
|
|
}
|
|
|
|
public function testHandleOptionGroupAndOption()
|
|
{
|
|
$html = sprintf(self::HTML_STUB, '<optgroup id="foo" label="foo" ><option value="foo">bar</option></optgroup>');
|
|
$doc = $this->parse($html);
|
|
|
|
$list = $doc->getElementById('foo');
|
|
|
|
$this->assertEquals(1, $list->childNodes->length);
|
|
|
|
$option = $list->childNodes->item(0);
|
|
$this->assertEquals('option', $option->tagName);
|
|
}
|
|
|
|
public function testTable()
|
|
{
|
|
$html = sprintf(self::HTML_STUB, '<table><thead id="a"><th>foo<td>bar<td>baz');
|
|
$doc = $this->parse($html);
|
|
|
|
$list = $doc->getElementById('a');
|
|
|
|
$this->assertEquals(3, $list->childNodes->length);
|
|
$this->assertEquals('th', $list->firstChild->tagName);
|
|
$this->assertEquals('td', $list->lastChild->tagName);
|
|
}
|
|
}
|