Files
ttrss-plugin-af-fulltext/vendor/masterminds/html5/test/HTML5/Parser/ScannerTest.php
Michal 9bc854ec44 af_fulltext: rule-driven extraction with a pluggable renderer
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs
go back to being real feed URLs and extraction happens inside tt-rss, using the
same ftr-site-config rules the proxy used.

The engine in lib/ has no tt-rss dependencies, so rules can be developed and
audited from the command line; init.php is a thin adapter over it.

Two findings from measuring the real subscription first, both of which shaped
the design:

- Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of
  chrome on a Cloudflare post), but it is an excellent renderer. So it is used
  for rawHtml only and the rule engine does the extraction.
- A body rule that stops matching after a redesign falls through to Readability
  and still produces a plausible article, so the breakage is invisible. Every
  extraction now records which rule matched and whether it fell back; auditing
  the 34 live feeds surfaced five community rules that match nothing.

Custom rules included for the sites that needed them, including three comics
where the article is an image and text-scoring extractors return the wrong thing
or nothing at all.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
2026-08-25 00:16:16 +01:00

185 lines
4.4 KiB
PHP

<?php
/**
* @file
* Test the Scanner. This requires the InputStream tests are all good.
*/
namespace Masterminds\HTML5\Tests\Parser;
use Masterminds\HTML5\Parser\StringInputStream;
use Masterminds\HTML5\Parser\Scanner;
class ScannerTest extends \Masterminds\HTML5\Tests\TestCase
{
/**
* A canary test to make sure the basics are setup and working.
*/
public function testConstructDeprecated()
{
$is = new StringInputStream('abc');
$s = new Scanner($is);
$this->assertInstanceOf('\Masterminds\HTML5\Parser\Scanner', $s);
}
public function testConstruct()
{
$this->assertInstanceOf('\Masterminds\HTML5\Parser\Scanner', new Scanner('abc'));
}
public function testNextDeprecated()
{
$s = new Scanner(new StringInputStream('abc'));
$this->assertEquals('b', $s->next());
$this->assertEquals('c', $s->next());
}
public function testNext()
{
$s = new Scanner('abc');
$this->assertEquals('b', $s->next());
$this->assertEquals('c', $s->next());
}
public function testPosition()
{
$s = new Scanner('abc');
$this->assertEquals(0, $s->position());
$s->next();
$this->assertEquals(1, $s->position());
}
public function testPeek()
{
$s = new Scanner('abc');
$this->assertEquals('b', $s->peek());
$s->next();
$this->assertEquals('c', $s->peek());
}
public function testCurrent()
{
$s = new Scanner('abc');
// Before scanning the string begins the current is empty.
$this->assertEquals('a', $s->current());
$c = $s->next();
$this->assertEquals('b', $s->current());
// Test movement through the string.
$c = $s->next();
$this->assertEquals('c', $s->current());
}
public function testUnconsume()
{
$s = new Scanner('abcdefghijklmnopqrst');
// Get initial position.
$s->next();
$start = $s->position();
// Move forward a bunch of positions.
$amount = 7;
for ($i = 0; $i < $amount; ++$i) {
$s->next();
}
// Roll back the amount we moved forward.
$s->unconsume($amount);
$this->assertEquals($start, $s->position());
}
public function testGetHex()
{
$s = new Scanner('ab13ck45DE*');
$this->assertEquals('ab13c', $s->getHex());
$s->next();
$this->assertEquals('45DE', $s->getHex());
}
public function testGetAsciiAlpha()
{
$s = new Scanner('abcdef1%mnop*');
$this->assertEquals('abcdef', $s->getAsciiAlpha());
// Move past the 1% to scan the next group of text.
$s->next();
$s->next();
$this->assertEquals('mnop', $s->getAsciiAlpha());
}
public function testGetAsciiAlphaNum()
{
$s = new Scanner('abcdef1ghpo#mn94op');
$this->assertEquals('abcdef1ghpo', $s->getAsciiAlphaNum());
// Move past the # to scan the next group of text.
$s->next();
$this->assertEquals('mn94op', $s->getAsciiAlphaNum());
}
public function testGetNumeric()
{
$s = new Scanner('1784a 45 9867 #');
$this->assertEquals('1784', $s->getNumeric());
// Move past the 'a ' to scan the next group of text.
$s->next();
$s->next();
$this->assertEquals('45', $s->getNumeric());
}
public function testCurrentLine()
{
$s = new Scanner("1784a\n45\n9867 #\nThis is a test.");
$this->assertEquals(1, $s->currentLine());
// Move to the next line.
$s->getAsciiAlphaNum();
$s->next();
$this->assertEquals(2, $s->currentLine());
}
public function testColumnOffset()
{
$s = new Scanner("1784a a\n45 9867 #\nThis is a test.");
// Move the pointer to the space.
$s->getAsciiAlphaNum();
$this->assertEquals(5, $s->columnOffset());
// We move the pointer ahead. There must be a better way to do this.
$s->next();
$s->next();
$s->next();
$s->next();
$s->next();
$s->next();
$this->assertEquals(3, $s->columnOffset());
}
public function testRemainingChars()
{
$string = "\n45\n9867 #\nThis is a test.";
$s = new Scanner("1784a\n45\n9867 #\nThis is a test.");
$s->getAsciiAlphaNum();
$this->assertEquals($string, $s->remainingChars());
}
}