feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed

src/ContentFeed/HtmlToMarkdownConverter.php

@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\ContentFeed;
+
+use League\HTMLToMarkdown\Converter\TableConverter;
+use League\HTMLToMarkdown\HtmlConverter;
+use Symfony\Component\DomCrawler\UriResolver;
+
+/**
+ * Turns the CMS's HTML, record text fields and page fragments alike, into Markdown for the
+ * index and the tool output. LLMs read Markdown well; HTML noise only costs tokens.
+ *
+ * Embeds, media and forms are dropped with their content. Relative links are made
+ * absolute against the document's own URL, since the page fragment may carry them.
+ */
+class HtmlToMarkdownConverter
+{
+    private const REMOVED_ELEMENTS = ['script', 'style', 'noscript', 'iframe', 'img', 'picture', 'svg', 'form'];
+
+    private readonly HtmlConverter $converter;
+
+    public function __construct()
+    {
+        $this->converter = new HtmlConverter([
+            'header_style' => 'atx',
+            'strip_tags' => true,
+            'strip_placeholder_links' => true,
+            'hard_break' => true,
+            'suppress_errors' => true,
+            'remove_nodes' => implode(' ', self::REMOVED_ELEMENTS),
+        ]);
+        $this->converter->getEnvironment()->addConverter(new TableConverter());
+    }
+
+    /**
+     * @param string|null $baseUrl what relative links resolve against; left as they are without one
+     */
+    public function convert(?string $html, ?string $baseUrl = null): string
+    {
+        if (null === $html || '' === trim($html)) {
+            return '';
+        }
+
+        $markdown = $this->converter->convert($this->prepare($html, $baseUrl));
+
+        return $this->collapseWhitespace($markdown);
+    }
+
+    private function prepare(string $html, ?string $baseUrl): string
+    {
+        $document = \Dom\HTMLDocument::createFromString('<!DOCTYPE html><html><body>'.$html.'</body></html>', \LIBXML_NOERROR, 'UTF-8');
+        $body = $document->body;
+
+        if (null === $body) {
+            return $html;
+        }
+
+        foreach ($body->querySelectorAll(implode(',', self::REMOVED_ELEMENTS)) as $element) {
+            $element->remove();
+        }
+
+        // Whitespace in text collapses as a browser would; the template's indentation would
+        // otherwise end up inside link labels and headings.
+        foreach ((new \Dom\XPath($document))->query('//body//text()[not(ancestor::pre)]') as $text) {
+            if (!$text instanceof \Dom\Text) {
+                continue;
+            }
+            $text->data = (string) preg_replace('/[ \t\r\n]+/', ' ', $text->data);
+        }
+
+        if (null !== $baseUrl) {
+            foreach ($body->querySelectorAll('a[href]') as $link) {
+                $href = trim((string) $link->getAttribute('href'));
+
+                if ('' === $href || preg_match('/^(mailto|tel|javascript):/i', $href)) {
+                    continue;
+                }
+
+                $link->setAttribute('href', UriResolver::resolve($href, $baseUrl));
+            }
+        }
+
+        return $body->innerHTML;
+    }
+
+    private function collapseWhitespace(string $markdown): string
+    {
+        // The converter leaves text entity-encoded (`&amp;`), which only costs tokens.
+        $markdown = html_entity_decode($markdown, \ENT_QUOTES | \ENT_HTML5, 'UTF-8');
+        $markdown = str_replace(["\r\n", "\r", "\u{00A0}"], ["\n", "\n", ' '], $markdown);
+
+        // Labels padded by the collapsed whitespace: "[ Kontakt ](…)" -> "[Kontakt](…)".
+        $markdown = (string) preg_replace(['/\[ +/', '/ +\]\(/'], ['[', ']('], $markdown);
+
+        $lines = array_map(
+            // Inner runs only: leading spaces carry list nesting.
+            static fn (string $line): string => rtrim((string) preg_replace('/(?<=\S)[ \t]{2,}/', ' ', $line)),
+            explode("\n", $markdown),
+        );
+
+        return trim((string) preg_replace("/\n{3,}/", "\n\n", implode("\n", $lines)));
+    }
+}