106 lines
3.8 KiB
PHP
106 lines
3.8 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\ContentFeed;
|
|
|
|
use League\HTMLToMarkdown\Converter\TableConverter;
|
|
use League\HTMLToMarkdown\HtmlConverter;
|
|
use Symfony\Component\DomCrawler\UriResolver;
|
|
|
|
/**
|
|
* Turns the CMS's HTML, record text fields and page fragments alike, into Markdown for the
|
|
* index and the tool output. LLMs read Markdown well; HTML noise only costs tokens.
|
|
*
|
|
* Embeds, media and forms are dropped with their content. Relative links are made
|
|
* absolute against the document's own URL, since the page fragment may carry them.
|
|
*/
|
|
class HtmlToMarkdownConverter
|
|
{
|
|
private const REMOVED_ELEMENTS = ['script', 'style', 'noscript', 'iframe', 'img', 'picture', 'svg', 'form'];
|
|
|
|
private readonly HtmlConverter $converter;
|
|
|
|
public function __construct()
|
|
{
|
|
$this->converter = new HtmlConverter([
|
|
'header_style' => 'atx',
|
|
'strip_tags' => true,
|
|
'strip_placeholder_links' => true,
|
|
'hard_break' => true,
|
|
'suppress_errors' => true,
|
|
'remove_nodes' => implode(' ', self::REMOVED_ELEMENTS),
|
|
]);
|
|
$this->converter->getEnvironment()->addConverter(new TableConverter());
|
|
}
|
|
|
|
/**
|
|
* @param string|null $baseUrl what relative links resolve against; left as they are without one
|
|
*/
|
|
public function convert(?string $html, ?string $baseUrl = null): string
|
|
{
|
|
if (null === $html || '' === trim($html)) {
|
|
return '';
|
|
}
|
|
|
|
$markdown = $this->converter->convert($this->prepare($html, $baseUrl));
|
|
|
|
return $this->collapseWhitespace($markdown);
|
|
}
|
|
|
|
private function prepare(string $html, ?string $baseUrl): string
|
|
{
|
|
$document = \Dom\HTMLDocument::createFromString('<!DOCTYPE html><html><body>'.$html.'</body></html>', \LIBXML_NOERROR, 'UTF-8');
|
|
$body = $document->body;
|
|
|
|
if (null === $body) {
|
|
return $html;
|
|
}
|
|
|
|
foreach ($body->querySelectorAll(implode(',', self::REMOVED_ELEMENTS)) as $element) {
|
|
$element->remove();
|
|
}
|
|
|
|
// Whitespace in text collapses as a browser would; the template's indentation would
|
|
// otherwise end up inside link labels and headings.
|
|
foreach ((new \Dom\XPath($document))->query('//body//text()[not(ancestor::pre)]') as $text) {
|
|
if (!$text instanceof \Dom\Text) {
|
|
continue;
|
|
}
|
|
$text->data = (string) preg_replace('/[ \t\r\n]+/', ' ', $text->data);
|
|
}
|
|
|
|
if (null !== $baseUrl) {
|
|
foreach ($body->querySelectorAll('a[href]') as $link) {
|
|
$href = trim((string) $link->getAttribute('href'));
|
|
|
|
if ('' === $href || preg_match('/^(mailto|tel|javascript):/i', $href)) {
|
|
continue;
|
|
}
|
|
|
|
$link->setAttribute('href', UriResolver::resolve($href, $baseUrl));
|
|
}
|
|
}
|
|
|
|
return $body->innerHTML;
|
|
}
|
|
|
|
private function collapseWhitespace(string $markdown): string
|
|
{
|
|
// The converter leaves text entity-encoded (`&`), which only costs tokens.
|
|
$markdown = html_entity_decode($markdown, \ENT_QUOTES | \ENT_HTML5, 'UTF-8');
|
|
$markdown = str_replace(["\r\n", "\r", "\u{00A0}"], ["\n", "\n", ' '], $markdown);
|
|
|
|
// Labels padded by the collapsed whitespace: "[ Kontakt ](…)" -> "[Kontakt](…)".
|
|
$markdown = (string) preg_replace(['/\[ +/', '/ +\]\(/'], ['[', ']('], $markdown);
|
|
|
|
$lines = array_map(
|
|
// Inner runs only: leading spaces carry list nesting.
|
|
static fn (string $line): string => rtrim((string) preg_replace('/(?<=\S)[ \t]{2,}/', ' ', $line)),
|
|
explode("\n", $markdown),
|
|
);
|
|
|
|
return trim((string) preg_replace("/\n{3,}/", "\n\n", implode("\n", $lines)));
|
|
}
|
|
}
|