af_fulltext: rule-driven extraction with a pluggable renderer
Replaces subscribing feeds through a self-hosted Full-Text RSS proxy. Feed URLs go back to being real feed URLs and extraction happens inside tt-rss, using the same ftr-site-config rules the proxy used. The engine in lib/ has no tt-rss dependencies, so rules can be developed and audited from the command line; init.php is a thin adapter over it. Two findings from measuring the real subscription first, both of which shaped the design: - Firecrawl's own onlyMainContent is far too coarse to extract with (73KB of chrome on a Cloudflare post), but it is an excellent renderer. So it is used for rawHtml only and the rule engine does the extraction. - A body rule that stops matching after a redesign falls through to Readability and still produces a plausible article, so the breakage is invisible. Every extraction now records which rule matched and whether it fell back; auditing the 34 live feeds surfaced five community rules that match nothing. Custom rules included for the sites that needed them, including three comics where the article is an image and text-scoring extractors return the wrong thing or nothing at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011sSJdftQx5bW5HZHgUKF3i
This commit is contained in:
Michal
18
vendor/psr/log/Psr/Log/Test/DummyTest.php
vendored
Normal file
18
vendor/psr/log/Psr/Log/Test/DummyTest.php
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
<?php
|
||||
|
||||
namespace Psr\Log\Test;
|
||||
|
||||
/**
|
||||
* This class is internal and does not follow the BC promise.
|
||||
*
|
||||
* Do NOT use this class in any way.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
class DummyTest
|
||||
{
|
||||
public function __toString()
|
||||
{
|
||||
return 'DummyTest';
|
||||
}
|
||||
}
|
||||
138
vendor/psr/log/Psr/Log/Test/LoggerInterfaceTest.php
vendored
Normal file
138
vendor/psr/log/Psr/Log/Test/LoggerInterfaceTest.php
vendored
Normal file
@@ -0,0 +1,138 @@
|
||||
<?php
|
||||
|
||||
namespace Psr\Log\Test;
|
||||
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Psr\Log\LogLevel;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/**
|
||||
* Provides a base test class for ensuring compliance with the LoggerInterface.
|
||||
*
|
||||
* Implementors can extend the class and implement abstract methods to run this
|
||||
* as part of their test suite.
|
||||
*/
|
||||
abstract class LoggerInterfaceTest extends TestCase
|
||||
{
|
||||
/**
|
||||
* @return LoggerInterface
|
||||
*/
|
||||
abstract public function getLogger();
|
||||
|
||||
/**
|
||||
* This must return the log messages in order.
|
||||
*
|
||||
* The simple formatting of the messages is: "<LOG LEVEL> <MESSAGE>".
|
||||
*
|
||||
* Example ->error('Foo') would yield "error Foo".
|
||||
*
|
||||
* @return string[]
|
||||
*/
|
||||
abstract public function getLogs();
|
||||
|
||||
public function testImplements()
|
||||
{
|
||||
$this->assertInstanceOf('Psr\Log\LoggerInterface', $this->getLogger());
|
||||
}
|
||||
|
||||
/**
|
||||
* @dataProvider provideLevelsAndMessages
|
||||
*/
|
||||
public function testLogsAtAllLevels($level, $message)
|
||||
{
|
||||
$logger = $this->getLogger();
|
||||
$logger->{$level}($message, array('user' => 'Bob'));
|
||||
$logger->log($level, $message, array('user' => 'Bob'));
|
||||
|
||||
$expected = array(
|
||||
$level.' message of level '.$level.' with context: Bob',
|
||||
$level.' message of level '.$level.' with context: Bob',
|
||||
);
|
||||
$this->assertEquals($expected, $this->getLogs());
|
||||
}
|
||||
|
||||
public function provideLevelsAndMessages()
|
||||
{
|
||||
return array(
|
||||
LogLevel::EMERGENCY => array(LogLevel::EMERGENCY, 'message of level emergency with context: {user}'),
|
||||
LogLevel::ALERT => array(LogLevel::ALERT, 'message of level alert with context: {user}'),
|
||||
LogLevel::CRITICAL => array(LogLevel::CRITICAL, 'message of level critical with context: {user}'),
|
||||
LogLevel::ERROR => array(LogLevel::ERROR, 'message of level error with context: {user}'),
|
||||
LogLevel::WARNING => array(LogLevel::WARNING, 'message of level warning with context: {user}'),
|
||||
LogLevel::NOTICE => array(LogLevel::NOTICE, 'message of level notice with context: {user}'),
|
||||
LogLevel::INFO => array(LogLevel::INFO, 'message of level info with context: {user}'),
|
||||
LogLevel::DEBUG => array(LogLevel::DEBUG, 'message of level debug with context: {user}'),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @expectedException \Psr\Log\InvalidArgumentException
|
||||
*/
|
||||
public function testThrowsOnInvalidLevel()
|
||||
{
|
||||
$logger = $this->getLogger();
|
||||
$logger->log('invalid level', 'Foo');
|
||||
}
|
||||
|
||||
public function testContextReplacement()
|
||||
{
|
||||
$logger = $this->getLogger();
|
||||
$logger->info('{Message {nothing} {user} {foo.bar} a}', array('user' => 'Bob', 'foo.bar' => 'Bar'));
|
||||
|
||||
$expected = array('info {Message {nothing} Bob Bar a}');
|
||||
$this->assertEquals($expected, $this->getLogs());
|
||||
}
|
||||
|
||||
public function testObjectCastToString()
|
||||
{
|
||||
if (method_exists($this, 'createPartialMock')) {
|
||||
$dummy = $this->createPartialMock('Psr\Log\Test\DummyTest', array('__toString'));
|
||||
} else {
|
||||
$dummy = $this->getMock('Psr\Log\Test\DummyTest', array('__toString'));
|
||||
}
|
||||
$dummy->expects($this->once())
|
||||
->method('__toString')
|
||||
->will($this->returnValue('DUMMY'));
|
||||
|
||||
$this->getLogger()->warning($dummy);
|
||||
|
||||
$expected = array('warning DUMMY');
|
||||
$this->assertEquals($expected, $this->getLogs());
|
||||
}
|
||||
|
||||
public function testContextCanContainAnything()
|
||||
{
|
||||
$closed = fopen('php://memory', 'r');
|
||||
fclose($closed);
|
||||
|
||||
$context = array(
|
||||
'bool' => true,
|
||||
'null' => null,
|
||||
'string' => 'Foo',
|
||||
'int' => 0,
|
||||
'float' => 0.5,
|
||||
'nested' => array('with object' => new DummyTest),
|
||||
'object' => new \DateTime,
|
||||
'resource' => fopen('php://memory', 'r'),
|
||||
'closed' => $closed,
|
||||
);
|
||||
|
||||
$this->getLogger()->warning('Crazy context data', $context);
|
||||
|
||||
$expected = array('warning Crazy context data');
|
||||
$this->assertEquals($expected, $this->getLogs());
|
||||
}
|
||||
|
||||
public function testContextExceptionKeyCanBeExceptionOrOtherValues()
|
||||
{
|
||||
$logger = $this->getLogger();
|
||||
$logger->warning('Random message', array('exception' => 'oops'));
|
||||
$logger->critical('Uncaught Exception!', array('exception' => new \LogicException('Fail')));
|
||||
|
||||
$expected = array(
|
||||
'warning Random message',
|
||||
'critical Uncaught Exception!'
|
||||
);
|
||||
$this->assertEquals($expected, $this->getLogs());
|
||||
}
|
||||
}
|
||||
147
vendor/psr/log/Psr/Log/Test/TestLogger.php
vendored
Normal file
147
vendor/psr/log/Psr/Log/Test/TestLogger.php
vendored
Normal file
@@ -0,0 +1,147 @@
|
||||
<?php
|
||||
|
||||
namespace Psr\Log\Test;
|
||||
|
||||
use Psr\Log\AbstractLogger;
|
||||
|
||||
/**
|
||||
* Used for testing purposes.
|
||||
*
|
||||
* It records all records and gives you access to them for verification.
|
||||
*
|
||||
* @method bool hasEmergency($record)
|
||||
* @method bool hasAlert($record)
|
||||
* @method bool hasCritical($record)
|
||||
* @method bool hasError($record)
|
||||
* @method bool hasWarning($record)
|
||||
* @method bool hasNotice($record)
|
||||
* @method bool hasInfo($record)
|
||||
* @method bool hasDebug($record)
|
||||
*
|
||||
* @method bool hasEmergencyRecords()
|
||||
* @method bool hasAlertRecords()
|
||||
* @method bool hasCriticalRecords()
|
||||
* @method bool hasErrorRecords()
|
||||
* @method bool hasWarningRecords()
|
||||
* @method bool hasNoticeRecords()
|
||||
* @method bool hasInfoRecords()
|
||||
* @method bool hasDebugRecords()
|
||||
*
|
||||
* @method bool hasEmergencyThatContains($message)
|
||||
* @method bool hasAlertThatContains($message)
|
||||
* @method bool hasCriticalThatContains($message)
|
||||
* @method bool hasErrorThatContains($message)
|
||||
* @method bool hasWarningThatContains($message)
|
||||
* @method bool hasNoticeThatContains($message)
|
||||
* @method bool hasInfoThatContains($message)
|
||||
* @method bool hasDebugThatContains($message)
|
||||
*
|
||||
* @method bool hasEmergencyThatMatches($message)
|
||||
* @method bool hasAlertThatMatches($message)
|
||||
* @method bool hasCriticalThatMatches($message)
|
||||
* @method bool hasErrorThatMatches($message)
|
||||
* @method bool hasWarningThatMatches($message)
|
||||
* @method bool hasNoticeThatMatches($message)
|
||||
* @method bool hasInfoThatMatches($message)
|
||||
* @method bool hasDebugThatMatches($message)
|
||||
*
|
||||
* @method bool hasEmergencyThatPasses($message)
|
||||
* @method bool hasAlertThatPasses($message)
|
||||
* @method bool hasCriticalThatPasses($message)
|
||||
* @method bool hasErrorThatPasses($message)
|
||||
* @method bool hasWarningThatPasses($message)
|
||||
* @method bool hasNoticeThatPasses($message)
|
||||
* @method bool hasInfoThatPasses($message)
|
||||
* @method bool hasDebugThatPasses($message)
|
||||
*/
|
||||
class TestLogger extends AbstractLogger
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
public $records = [];
|
||||
|
||||
public $recordsByLevel = [];
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function log($level, $message, array $context = [])
|
||||
{
|
||||
$record = [
|
||||
'level' => $level,
|
||||
'message' => $message,
|
||||
'context' => $context,
|
||||
];
|
||||
|
||||
$this->recordsByLevel[$record['level']][] = $record;
|
||||
$this->records[] = $record;
|
||||
}
|
||||
|
||||
public function hasRecords($level)
|
||||
{
|
||||
return isset($this->recordsByLevel[$level]);
|
||||
}
|
||||
|
||||
public function hasRecord($record, $level)
|
||||
{
|
||||
if (is_string($record)) {
|
||||
$record = ['message' => $record];
|
||||
}
|
||||
return $this->hasRecordThatPasses(function ($rec) use ($record) {
|
||||
if ($rec['message'] !== $record['message']) {
|
||||
return false;
|
||||
}
|
||||
if (isset($record['context']) && $rec['context'] !== $record['context']) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}, $level);
|
||||
}
|
||||
|
||||
public function hasRecordThatContains($message, $level)
|
||||
{
|
||||
return $this->hasRecordThatPasses(function ($rec) use ($message) {
|
||||
return strpos($rec['message'], $message) !== false;
|
||||
}, $level);
|
||||
}
|
||||
|
||||
public function hasRecordThatMatches($regex, $level)
|
||||
{
|
||||
return $this->hasRecordThatPasses(function ($rec) use ($regex) {
|
||||
return preg_match($regex, $rec['message']) > 0;
|
||||
}, $level);
|
||||
}
|
||||
|
||||
public function hasRecordThatPasses(callable $predicate, $level)
|
||||
{
|
||||
if (!isset($this->recordsByLevel[$level])) {
|
||||
return false;
|
||||
}
|
||||
foreach ($this->recordsByLevel[$level] as $i => $rec) {
|
||||
if (call_user_func($predicate, $rec, $i)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function __call($method, $args)
|
||||
{
|
||||
if (preg_match('/(.*)(Debug|Info|Notice|Warning|Error|Critical|Alert|Emergency)(.*)/', $method, $matches) > 0) {
|
||||
$genericMethod = $matches[1] . ('Records' !== $matches[3] ? 'Record' : '') . $matches[3];
|
||||
$level = strtolower($matches[2]);
|
||||
if (method_exists($this, $genericMethod)) {
|
||||
$args[] = $level;
|
||||
return call_user_func_array([$this, $genericMethod], $args);
|
||||
}
|
||||
}
|
||||
throw new \BadMethodCallException('Call to undefined method ' . get_class($this) . '::' . $method . '()');
|
||||
}
|
||||
|
||||
public function reset()
|
||||
{
|
||||
$this->records = [];
|
||||
$this->recordsByLevel = [];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user