61 lines
2.1 KiB
PHP
61 lines
2.1 KiB
PHP
|
#!/usr/bin/env php
|
||
|
|
<?php
|
||
|
|
/**
|
||
|
|
* Standalone harness for the extraction engine.
|
||
|
|
*
|
||
|
|
* The whole point of keeping lib/ free of tt-rss dependencies: rules can be
|
||
|
|
* written and checked against real URLs without a running instance.
|
||
|
|
*
|
||
|
|
* php bin/extract.php <url> [--backend=direct|firecrawl] [--rule=FILE] [--html] [--quiet]
|
||
|
|
*/
|
||
|
|
require_once __DIR__ . '/../autoload.php';
|
||
|
|
|
||
|
|
use AfFulltext\{CurlFetcher, Extractor, FirecrawlFetcher, RuleSet};
|
||
|
|
|
||
|
|
$args = array_slice($argv, 1);
|
||
|
|
$opts = ['backend' => 'direct', 'rule' => null, 'html' => false, 'quiet' => false];
|
||
|
|
$url = null;
|
||
|
|
|
||
|
|
foreach ($args as $arg) {
|
||
|
|
if (str_starts_with($arg, '--backend=')) $opts['backend'] = substr($arg, 10);
|
||
|
|
elseif (str_starts_with($arg, '--rule=')) $opts['rule'] = substr($arg, 7);
|
||
|
|
elseif ($arg === '--html') $opts['html'] = true;
|
||
|
|
elseif ($arg === '--quiet') $opts['quiet'] = true;
|
||
|
|
elseif (!str_starts_with($arg, '--')) $url = $arg;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!$url) {
|
||
|
|
fwrite(STDERR, "usage: extract.php <url> [--backend=direct|firecrawl] [--rule=FILE] [--html] [--quiet]\n");
|
||
|
|
exit(2);
|
||
|
|
}
|
||
|
|
|
||
|
|
$base = dirname(__DIR__);
|
||
|
|
$dirs = [$base . '/site_config/custom', $base . '/site_config/standard'];
|
||
|
|
|
||
|
|
// --rule points at a single file to try, overriding whatever the site config says.
|
||
|
|
if ($opts['rule']) {
|
||
|
|
$tmp = sys_get_temp_dir() . '/af_fulltext_rule_' . getmypid();
|
||
|
|
@mkdir($tmp, 0700, true);
|
||
|
|
$host = strtolower((string) parse_url($url, PHP_URL_HOST));
|
||
|
|
copy($opts['rule'], "$tmp/$host.txt");
|
||
|
|
array_unshift($dirs, $tmp);
|
||
|
|
}
|
||
|
|
|
||
|
|
$fetcher = $opts['backend'] === 'firecrawl'
|
||
|
|
? new FirecrawlFetcher(getenv('FIRECRAWL_URL') ?: 'http://127.0.0.1:13002')
|
||
|
|
: new CurlFetcher();
|
||
|
|
|
||
|
|
$result = (new Extractor(new RuleSet($dirs)))->extract($url, $fetcher);
|
||
|
|
|
||
|
|
if (!$opts['quiet']) {
|
||
|
|
fwrite(STDERR, $result->summary() . "\n");
|
||
|
|
foreach ($result->rule_sources as $s) fwrite(STDERR, " rule: $s\n");
|
||
|
|
foreach ($result->errors as $e) fwrite(STDERR, " error: $e\n");
|
||
|
|
if ($result->title) fwrite(STDERR, " title: {$result->title}\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
if ($opts['html']) echo $result->html, "\n";
|
||
|
|
else echo trim(preg_replace('/\s+/u', ' ', strip_tags($result->html)) ?? ''), "\n";
|
||
|
|
|
||
|
|
exit($result->ok() ? 0 : 1);
|