feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed

src/Chatbot/Search/ContentSearch.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Chatbot\Search;
+
+use App\Repository\ContentDocumentRepository;
+
+/**
+ * Full-text search over the mirrored website content.
+ *
+ * Ranked by relevance, weighted by type in the order people need them: what they book
+ * (products, hotels), where they go (destinations, concepts), general pages, then blog
+ * posts. The weights are multipliers on the relevance score, so a much better match of a
+ * lower type still wins.
+ */
+class ContentSearch
+{
+    /**
+     * The tool's `type` values. `destination` covers region, country and city.
+     */
+    public const TYPES = [
+        'page' => ['page'],
+        'product' => ['product'],
+        'hotel' => ['hotel'],
+        'destination' => ['region', 'country', 'city'],
+        'concept' => ['concept'],
+        'news' => ['news'],
+    ];
+
+    private const TYPE_WEIGHTS = [
+        'product' => 1.4,
+        'hotel' => 1.4,
+        'region' => 1.2,
+        'country' => 1.2,
+        'city' => 1.2,
+        'concept' => 1.2,
+        'page' => 1.0,
+        'news' => 0.8,
+    ];
+
+    public function __construct(private readonly ContentDocumentRepository $repository)
+    {
+    }
+
+    /**
+     * @param string|null $type a key of TYPES
+     *
+     * @return list<array{type: string, title: string, url: string, site: ?string, breadcrumb: ?string, excerpt: string}>
+     */
+    public function search(string $query, ?string $type, ?string $site, int $limit): array
+    {
+        $booleanQuery = ContentSearchQuery::booleanQuery($query);
+
+        if (null === $booleanQuery) {
+            return [];
+        }
+
+        $rows = $this->repository->searchFulltext($booleanQuery, null === $type ? [] : self::TYPES[$type], $site, self::TYPE_WEIGHTS, $limit);
+        $terms = ContentSearchQuery::terms($query);
+
+        return array_map(static fn (array $row): array => [
+            'type' => self::publicType($row['type']),
+            'title' => $row['title'],
+            'url' => $row['url'],
+            'site' => $row['site'],
+            'breadcrumb' => $row['breadcrumb'],
+            'excerpt' => Excerpt::around($row['body_md'], $terms),
+        ], $rows);
+    }
+
+    /**
+     * The tool's name for a stored type.
+     */
+    public static function publicType(string $type): string
+    {
+        foreach (self::TYPES as $public => $types) {
+            if (in_array($type, $types, true)) {
+                return $public;
+            }
+        }
+
+        return $type;
+    }
+}

src/Chatbot/Search/ContentSearchQuery.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Chatbot\Search;
+
+/**
+ * Turns free text into a MySQL boolean-mode full-text query.
+ *
+ * InnoDB has no German stemming, so every term becomes a prefix match (`skipass*` finds
+ * Skipässe no better, but finds Skipassversicherung). Terms are optional rather than
+ * required: a question rarely uses the page's own words for everything, and the relevance
+ * score already ranks documents matching more terms higher. German stopwords are dropped
+ * here, since the server's stopword list is English.
+ */
+final class ContentSearchQuery
+{
+    /** InnoDB's default innodb_ft_min_token_size; shorter tokens are not indexed. */
+    private const MIN_TOKEN_LENGTH = 3;
+    private const MAX_TERMS = 12;
+
+    private const STOPWORDS = [
+        'aber', 'alle', 'allem', 'allen', 'aller', 'alles', 'als', 'also', 'am', 'an', 'ander', 'andere', 'anderen',
+        'auch', 'auf', 'aus', 'bei', 'beim', 'bin', 'bis', 'bist', 'da', 'damit', 'dann', 'das', 'dass', 'dein',
+        'deine', 'dem', 'den', 'denn', 'der', 'des', 'dessen', 'die', 'dies', 'diese', 'diesem', 'diesen', 'dieser',
+        'dieses', 'doch', 'dort', 'du', 'durch', 'ein', 'eine', 'einem', 'einen', 'einer', 'eines', 'einige', 'er',
+        'es', 'etwas', 'euch', 'euer', 'eure', 'für', 'gibt', 'gib', 'hab', 'habe', 'haben', 'hast', 'hat', 'hatte',
+        'hier', 'hin', 'ich', 'ihr', 'ihre', 'ihrem', 'ihren', 'ihrer', 'im', 'in', 'ins', 'ist', 'ja', 'jede',
+        'jedem', 'jeden', 'jeder', 'jetzt', 'kann', 'kannst', 'kein', 'keine', 'können', 'könnt', 'man', 'mein',
+        'meine', 'mich', 'mir', 'mit', 'muss', 'musst', 'nach', 'nicht', 'nichts', 'noch', 'nun', 'nur', 'ob', 'oder',
+        'ohne', 'sehr', 'sein', 'seine', 'sich', 'sie', 'sind', 'so', 'soll', 'sollte', 'über', 'um', 'und', 'uns',
+        'unser', 'unsere', 'unter', 'viel', 'vom', 'von', 'vor', 'war', 'waren', 'was', 'weil', 'welche', 'welchem',
+        'welchen', 'welcher', 'welches', 'wenn', 'wer', 'werde', 'werden', 'wie', 'wieder', 'will', 'wir', 'wird',
+        'wo', 'woher', 'wohin', 'zu', 'zum', 'zur', 'zwischen', 'bitte', 'gerne', 'euren', 'eurem', 'eurer',
+    ];
+
+    /**
+     * The terms of a query: lower-cased, stopwords and short tokens dropped, unique.
+     *
+     * @return list<string>
+     */
+    public static function terms(string $text): array
+    {
+        $tokens = preg_split('/[^\p{L}\p{N}]+/u', mb_strtolower($text), -1, \PREG_SPLIT_NO_EMPTY) ?: [];
+        $terms = [];
+
+        foreach ($tokens as $token) {
+            if (mb_strlen($token) < self::MIN_TOKEN_LENGTH || in_array($token, self::STOPWORDS, true)) {
+                continue;
+            }
+            $terms[$token] = true;
+        }
+
+        return array_slice(array_keys($terms), 0, self::MAX_TERMS);
+    }
+
+    /**
+     * The boolean-mode query, or null when nothing searchable is left.
+     */
+    public static function booleanQuery(string $text): ?string
+    {
+        $terms = self::terms($text);
+
+        return [] === $terms ? null : implode(' ', array_map(static fn (string $term): string => $term.'*', $terms));
+    }
+}

src/Chatbot/Search/Excerpt.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Chatbot\Search;
+
+/**
+ * Cuts a window of about LENGTH characters out of a document, around its first match.
+ *
+ * The excerpt keeps search results small; get_document fetches the rest.
+ */
+final class Excerpt
+{
+    public const LENGTH = 600;
+
+    /**
+     * @param list<string> $terms lower-cased, matched as prefixes like the full-text search
+     */
+    public static function around(string $text, array $terms, int $length = self::LENGTH): string
+    {
+        $text = trim((string) preg_replace('/\s+/u', ' ', $text));
+        $textLength = mb_strlen($text);
+
+        if ($textLength <= $length) {
+            return $text;
+        }
+
+        $position = null;
+        $lower = mb_strtolower($text);
+        foreach ($terms as $term) {
+            $found = mb_strpos($lower, $term);
+            if (false !== $found && (null === $position || $found < $position)) {
+                $position = $found;
+            }
+        }
+
+        // Start a third of the window before the match, so it is read in context.
+        $start = max(0, min(($position ?? 0) - intdiv($length, 3), $textLength - $length));
+        $excerpt = mb_substr($text, $start, $length);
+
+        if ($start > 0) {
+            $space = mb_strpos($excerpt, ' ');
+            $excerpt = '…'.(false !== $space && $space < 40 ? mb_substr($excerpt, $space + 1) : $excerpt);
+        }
+
+        if ($start + $length < $textLength) {
+            $space = mb_strrpos($excerpt, ' ');
+            $excerpt = (false !== $space && mb_strlen($excerpt) - $space < 40 ? mb_substr($excerpt, 0, $space) : $excerpt).'…';
+        }
+
+        return $excerpt;
+    }
+}