feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed
This commit is contained in:
fromme committed
1 parent
ff7ddf7a9e
commit
1b4384e3f3
52 files changed
+5117
-4
No files matched your search
@@ -0,0 +1,16 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp;
|
||||
|
||||
/**
|
||||
* The site identifiers of the TYPO3 content feed, offered to the model as the `site` filter.
|
||||
* ep-events and unichamp are retired and not in the feed.
|
||||
*/
|
||||
final class Sites
|
||||
{
|
||||
public const IDENTIFIERS = ['ep-reisen', 'ser', 'ski-boarderweek', 'ski-wochenenden', 'snowzone'];
|
||||
|
||||
public const DESCRIPTION = 'Restrict to one website: ep-reisen (ep-reisen.de), ser (semesterende-skireisen.de), ski-boarderweek, ski-wochenenden, snowzone (snowzone.net).';
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp\Tool;
|
||||
|
||||
use App\Chatbot\Mcp\ToolCallRunner;
|
||||
use App\Chatbot\Search\ContentSearch;
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
use Mcp\Capability\Attribute\McpTool;
|
||||
use Mcp\Capability\Attribute\Schema;
|
||||
use Mcp\Exception\ToolCallException;
|
||||
use Mcp\Schema\Result\CallToolResult;
|
||||
use Mcp\Schema\ToolAnnotations;
|
||||
|
||||
#[McpTool(
|
||||
name: 'get_document',
|
||||
description: 'Read the full text of one search_content hit, by its url.',
|
||||
annotations: new ToolAnnotations(readOnlyHint: true, openWorldHint: false),
|
||||
)]
|
||||
final class GetDocumentTool
|
||||
{
|
||||
private const BODY_LIMIT = 20000;
|
||||
|
||||
public function __construct(
|
||||
private readonly ContentDocumentRepository $repository,
|
||||
private readonly ToolCallRunner $runner,
|
||||
) {
|
||||
}
|
||||
|
||||
public function __invoke(
|
||||
#[Schema(description: 'A url returned by search_content.', minLength: 1)]
|
||||
string $url,
|
||||
): CallToolResult {
|
||||
return $this->runner->run('get_document', get_defined_vars(), function () use ($url): array {
|
||||
$document = $this->repository->findOneIndexedByUrl(trim($url));
|
||||
|
||||
if (null === $document) {
|
||||
throw new ToolCallException('No document at this url. Use a url returned by search_content.');
|
||||
}
|
||||
|
||||
$body = $document->getBodyMd();
|
||||
$truncated = mb_strlen($body) > self::BODY_LIMIT;
|
||||
|
||||
return [
|
||||
'type' => ContentSearch::publicType($document->getType()),
|
||||
'title' => $document->getTitle(),
|
||||
'url' => $document->getUrl(),
|
||||
'site' => $document->getSite(),
|
||||
'breadcrumb' => $document->getBreadcrumb(),
|
||||
'body' => $truncated ? mb_substr($body, 0, self::BODY_LIMIT) : $body,
|
||||
'truncated' => $truncated,
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp\Tool;
|
||||
|
||||
use App\Chatbot\Mcp\ToolCallRunner;
|
||||
use App\Chatbot\TripCatalog;
|
||||
use Mcp\Capability\Attribute\McpTool;
|
||||
use Mcp\Capability\Attribute\Schema;
|
||||
use Mcp\Exception\ToolCallException;
|
||||
use Mcp\Schema\Result\CallToolResult;
|
||||
use Mcp\Schema\ToolAnnotations;
|
||||
|
||||
#[McpTool(
|
||||
name: 'get_trip',
|
||||
description: 'Full details of one trip: description, included services, programme, hotel, all upcoming dates with rooms, prices per person in EUR and availability, pickup points and departure times, FAQs. Use after search_trips, or when the user names a specific trip. Prices and availability are a snapshot as of dataAsOf; the booking page is binding.',
|
||||
annotations: new ToolAnnotations(readOnlyHint: true, openWorldHint: false),
|
||||
)]
|
||||
final class GetTripTool
|
||||
{
|
||||
public function __construct(
|
||||
private readonly TripCatalog $catalog,
|
||||
private readonly ToolCallRunner $runner,
|
||||
) {
|
||||
}
|
||||
|
||||
public function __invoke(
|
||||
#[Schema(description: 'Trip code, as returned by search_trips.', minLength: 1)]
|
||||
string $code,
|
||||
#[Schema(description: 'Narrows the dates to one hotel when a trip has several.')]
|
||||
?string $hotelCode = null,
|
||||
): CallToolResult {
|
||||
return $this->runner->run('get_trip', get_defined_vars(), function () use ($code, $hotelCode): array {
|
||||
$trip = $this->catalog->get(trim($code), $hotelCode);
|
||||
|
||||
if (null === $trip) {
|
||||
throw new ToolCallException(sprintf('No trip with code "%s". Use search_trips to find trip codes.', $code));
|
||||
}
|
||||
|
||||
return $trip;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp\Tool;
|
||||
|
||||
use App\Chatbot\Mcp\Sites;
|
||||
use App\Chatbot\Mcp\ToolCallRunner;
|
||||
use App\Chatbot\Search\ContentSearch;
|
||||
use Mcp\Capability\Attribute\McpTool;
|
||||
use Mcp\Capability\Attribute\Schema;
|
||||
use Mcp\Exception\ToolCallException;
|
||||
use Mcp\Schema\Result\CallToolResult;
|
||||
use Mcp\Schema\ToolAnnotations;
|
||||
|
||||
#[McpTool(
|
||||
name: 'search_content',
|
||||
description: "Search the websites' editorial content: destinations, hotels, travel information, FAQs, terms, blog articles, general company information. Use for questions that are not about specific dates or prices. Returns excerpts; use get_document for the full text of a hit.",
|
||||
annotations: new ToolAnnotations(readOnlyHint: true, openWorldHint: false),
|
||||
)]
|
||||
final class SearchContentTool
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ContentSearch $search,
|
||||
private readonly ToolCallRunner $runner,
|
||||
) {
|
||||
}
|
||||
|
||||
public function __invoke(
|
||||
#[Schema(description: 'What to look for, in German keywords works best, e.g. "Gepäck Snowboard Bus".', minLength: 1)]
|
||||
string $query,
|
||||
#[Schema(description: 'Restrict to one kind of content. destination = region, country or town.', enum: ['page', 'product', 'hotel', 'destination', 'concept', 'news'])]
|
||||
?string $type = null,
|
||||
#[Schema(description: Sites::DESCRIPTION, enum: Sites::IDENTIFIERS)]
|
||||
?string $site = null,
|
||||
#[Schema(description: 'Number of hits, default 5, at most 8.', minimum: 1, maximum: 8)]
|
||||
?int $limit = null,
|
||||
): CallToolResult {
|
||||
return $this->runner->run('search_content', get_defined_vars(), function () use ($query, $type, $site, $limit): array {
|
||||
if ('' === trim($query)) {
|
||||
throw new ToolCallException('query must not be empty.');
|
||||
}
|
||||
|
||||
return ['results' => $this->search->search($query, $type, $site, max(1, min(8, $limit ?? 5)))];
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp\Tool;
|
||||
|
||||
use App\Chatbot\Mcp\Sites;
|
||||
use App\Chatbot\Mcp\ToolCallRunner;
|
||||
use App\Chatbot\TripCatalog;
|
||||
use App\Model\Chatbot\TripCriteria;
|
||||
use Mcp\Capability\Attribute\McpTool;
|
||||
use Mcp\Capability\Attribute\Schema;
|
||||
use Mcp\Exception\ToolCallException;
|
||||
use Mcp\Schema\Result\CallToolResult;
|
||||
use Mcp\Schema\ToolAnnotations;
|
||||
|
||||
#[McpTool(
|
||||
name: 'search_trips',
|
||||
description: 'Find bookable trips by structured criteria: travel period, destination, trip type, price, group size. Use for any question about which trips exist, when, where, or how much. Returns trips with their next dates and lowest prices per person in EUR. Prices and availability are a snapshot as of dataAsOf; the booking page is binding.',
|
||||
annotations: new ToolAnnotations(readOnlyHint: true, openWorldHint: false),
|
||||
)]
|
||||
final class SearchTripsTool
|
||||
{
|
||||
public function __construct(
|
||||
private readonly TripCatalog $catalog,
|
||||
private readonly ToolCallRunner $runner,
|
||||
) {
|
||||
}
|
||||
|
||||
public function __invoke(
|
||||
#[Schema(description: 'Free text matched against trip, hotel, destination and concept names and texts, e.g. "Val Thorens" or "Snowboardkurs".')]
|
||||
?string $query = null,
|
||||
#[Schema(description: 'Start of the travel period, YYYY-MM-DD. Trips overlapping the period match.', format: 'date')]
|
||||
?string $dateFrom = null,
|
||||
#[Schema(description: 'End of the travel period, YYYY-MM-DD.', format: 'date')]
|
||||
?string $dateTo = null,
|
||||
#[Schema(description: 'Country name, e.g. "Frankreich".')]
|
||||
?string $country = null,
|
||||
#[Schema(description: 'Region, ski area or town name, e.g. "Les 3 Vallées".')]
|
||||
?string $region = null,
|
||||
#[Schema(description: 'Trip concept code or name, e.g. ski, party, group.')]
|
||||
?string $concept = null,
|
||||
#[Schema(description: 'Maximum price per person in EUR.', minimum: 0)]
|
||||
?float $maxPrice = null,
|
||||
#[Schema(description: 'Only dates with at least this many places available.', minimum: 1)]
|
||||
?int $persons = null,
|
||||
#[Schema(description: 'true for day trips only, false to exclude them.')]
|
||||
?bool $daytrip = null,
|
||||
#[Schema(description: Sites::DESCRIPTION, enum: Sites::IDENTIFIERS)]
|
||||
?string $site = null,
|
||||
#[Schema(description: 'Number of trips, default 5, at most 10.', minimum: 1, maximum: 10)]
|
||||
?int $limit = null,
|
||||
): CallToolResult {
|
||||
return $this->runner->run('search_trips', get_defined_vars(), function () use ($query, $dateFrom, $dateTo, $country, $region, $concept, $maxPrice, $persons, $daytrip, $site, $limit): array {
|
||||
$from = $this->date($dateFrom, 'dateFrom');
|
||||
$to = $this->date($dateTo, 'dateTo');
|
||||
|
||||
if (null !== $from && null !== $to && $from > $to) {
|
||||
throw new ToolCallException('dateFrom must not be after dateTo.');
|
||||
}
|
||||
|
||||
return $this->catalog->search(new TripCriteria(
|
||||
query: $this->text($query),
|
||||
dateFrom: $from,
|
||||
dateTo: $to,
|
||||
country: $this->text($country),
|
||||
region: $this->text($region),
|
||||
concept: $this->text($concept),
|
||||
maxPrice: $maxPrice,
|
||||
persons: $persons,
|
||||
daytrip: $daytrip,
|
||||
site: $site,
|
||||
limit: max(1, min(10, $limit ?? 5)),
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
private function date(?string $value, string $name): ?\DateTimeImmutable
|
||||
{
|
||||
if (null === $value || '' === trim($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$date = \DateTimeImmutable::createFromFormat('!Y-m-d', trim($value));
|
||||
|
||||
if (false === $date || $date->format('Y-m-d') !== trim($value)) {
|
||||
throw new ToolCallException(sprintf('%s must be a date in the form YYYY-MM-DD.', $name));
|
||||
}
|
||||
|
||||
return $date;
|
||||
}
|
||||
|
||||
private function text(?string $value): ?string
|
||||
{
|
||||
return null === $value || '' === trim($value) ? null : trim($value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Mcp;
|
||||
|
||||
use Mcp\Exception\ToolCallException;
|
||||
use Mcp\Schema\Content\TextContent;
|
||||
use Mcp\Schema\Result\CallToolResult;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Bundle\SecurityBundle\Security;
|
||||
use Symfony\Component\RateLimiter\RateLimiterFactory;
|
||||
|
||||
/**
|
||||
* What every chatbot tool call goes through: the per-token rate limit, the call log, and
|
||||
* the answer as compact JSON.
|
||||
*
|
||||
* The SDK would pretty-print an array result and repeat it as structured content; both
|
||||
* cost the model tokens and neither helps n8n. The log records tool, arguments, result
|
||||
* count and duration, never anything about the person chatting, which n8n does not send.
|
||||
*/
|
||||
class ToolCallRunner
|
||||
{
|
||||
public function __construct(
|
||||
private readonly RateLimiterFactory $chatbotMcpLimiter,
|
||||
private readonly Security $security,
|
||||
private readonly LoggerInterface $chatbotLogger,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $arguments as received, for the log
|
||||
* @param callable(): array<string, mixed> $call
|
||||
*
|
||||
* @throws ToolCallException invalid input or rate limit; the model sees the message
|
||||
*/
|
||||
public function run(string $tool, array $arguments, callable $call): CallToolResult
|
||||
{
|
||||
$arguments = array_filter($arguments, static fn (mixed $value): bool => null !== $value);
|
||||
$limit = $this->chatbotMcpLimiter->create($this->security->getUser()?->getUserIdentifier() ?? 'anonymous')->consume();
|
||||
|
||||
if (!$limit->isAccepted()) {
|
||||
$this->chatbotLogger->warning('MCP tool call rate-limited', ['tool' => $tool, 'arguments' => $arguments]);
|
||||
|
||||
throw new ToolCallException(sprintf('Too many tool calls. Retry after %s.', $limit->getRetryAfter()->format('H:i:s')));
|
||||
}
|
||||
|
||||
$started = hrtime(true);
|
||||
|
||||
try {
|
||||
$result = $call();
|
||||
} catch (ToolCallException $e) {
|
||||
$this->chatbotLogger->info('MCP tool call rejected', ['tool' => $tool, 'arguments' => $arguments, 'error' => $e->getMessage()]);
|
||||
|
||||
throw $e;
|
||||
} catch (\Throwable $e) {
|
||||
$this->chatbotLogger->error('MCP tool call failed', ['tool' => $tool, 'arguments' => $arguments, 'exception' => $e]);
|
||||
|
||||
throw $e;
|
||||
}
|
||||
|
||||
$this->chatbotLogger->info('MCP tool call', [
|
||||
'tool' => $tool,
|
||||
'arguments' => $arguments,
|
||||
'resultCount' => is_array($result['results'] ?? null) ? count($result['results']) : 1,
|
||||
'durationMs' => (int) ((hrtime(true) - $started) / 1_000_000),
|
||||
]);
|
||||
|
||||
return new CallToolResult([new TextContent(json_encode($result, \JSON_UNESCAPED_SLASHES | \JSON_UNESCAPED_UNICODE | \JSON_THROW_ON_ERROR | \JSON_INVALID_UTF8_SUBSTITUTE))]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Search;
|
||||
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
|
||||
/**
|
||||
* Full-text search over the mirrored website content.
|
||||
*
|
||||
* Ranked by relevance, weighted by type in the order people need them: what they book
|
||||
* (products, hotels), where they go (destinations, concepts), general pages, then blog
|
||||
* posts. The weights are multipliers on the relevance score, so a much better match of a
|
||||
* lower type still wins.
|
||||
*/
|
||||
class ContentSearch
|
||||
{
|
||||
/**
|
||||
* The tool's `type` values. `destination` covers region, country and city.
|
||||
*/
|
||||
public const TYPES = [
|
||||
'page' => ['page'],
|
||||
'product' => ['product'],
|
||||
'hotel' => ['hotel'],
|
||||
'destination' => ['region', 'country', 'city'],
|
||||
'concept' => ['concept'],
|
||||
'news' => ['news'],
|
||||
];
|
||||
|
||||
private const TYPE_WEIGHTS = [
|
||||
'product' => 1.4,
|
||||
'hotel' => 1.4,
|
||||
'region' => 1.2,
|
||||
'country' => 1.2,
|
||||
'city' => 1.2,
|
||||
'concept' => 1.2,
|
||||
'page' => 1.0,
|
||||
'news' => 0.8,
|
||||
];
|
||||
|
||||
public function __construct(private readonly ContentDocumentRepository $repository)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $type a key of TYPES
|
||||
*
|
||||
* @return list<array{type: string, title: string, url: string, site: ?string, breadcrumb: ?string, excerpt: string}>
|
||||
*/
|
||||
public function search(string $query, ?string $type, ?string $site, int $limit): array
|
||||
{
|
||||
$booleanQuery = ContentSearchQuery::booleanQuery($query);
|
||||
|
||||
if (null === $booleanQuery) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$rows = $this->repository->searchFulltext($booleanQuery, null === $type ? [] : self::TYPES[$type], $site, self::TYPE_WEIGHTS, $limit);
|
||||
$terms = ContentSearchQuery::terms($query);
|
||||
|
||||
return array_map(static fn (array $row): array => [
|
||||
'type' => self::publicType($row['type']),
|
||||
'title' => $row['title'],
|
||||
'url' => $row['url'],
|
||||
'site' => $row['site'],
|
||||
'breadcrumb' => $row['breadcrumb'],
|
||||
'excerpt' => Excerpt::around($row['body_md'], $terms),
|
||||
], $rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* The tool's name for a stored type.
|
||||
*/
|
||||
public static function publicType(string $type): string
|
||||
{
|
||||
foreach (self::TYPES as $public => $types) {
|
||||
if (in_array($type, $types, true)) {
|
||||
return $public;
|
||||
}
|
||||
}
|
||||
|
||||
return $type;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Search;
|
||||
|
||||
/**
|
||||
* Turns free text into a MySQL boolean-mode full-text query.
|
||||
*
|
||||
* InnoDB has no German stemming, so every term becomes a prefix match (`skipass*` finds
|
||||
* Skipässe no better, but finds Skipassversicherung). Terms are optional rather than
|
||||
* required: a question rarely uses the page's own words for everything, and the relevance
|
||||
* score already ranks documents matching more terms higher. German stopwords are dropped
|
||||
* here, since the server's stopword list is English.
|
||||
*/
|
||||
final class ContentSearchQuery
|
||||
{
|
||||
/** InnoDB's default innodb_ft_min_token_size; shorter tokens are not indexed. */
|
||||
private const MIN_TOKEN_LENGTH = 3;
|
||||
private const MAX_TERMS = 12;
|
||||
|
||||
private const STOPWORDS = [
|
||||
'aber', 'alle', 'allem', 'allen', 'aller', 'alles', 'als', 'also', 'am', 'an', 'ander', 'andere', 'anderen',
|
||||
'auch', 'auf', 'aus', 'bei', 'beim', 'bin', 'bis', 'bist', 'da', 'damit', 'dann', 'das', 'dass', 'dein',
|
||||
'deine', 'dem', 'den', 'denn', 'der', 'des', 'dessen', 'die', 'dies', 'diese', 'diesem', 'diesen', 'dieser',
|
||||
'dieses', 'doch', 'dort', 'du', 'durch', 'ein', 'eine', 'einem', 'einen', 'einer', 'eines', 'einige', 'er',
|
||||
'es', 'etwas', 'euch', 'euer', 'eure', 'für', 'gibt', 'gib', 'hab', 'habe', 'haben', 'hast', 'hat', 'hatte',
|
||||
'hier', 'hin', 'ich', 'ihr', 'ihre', 'ihrem', 'ihren', 'ihrer', 'im', 'in', 'ins', 'ist', 'ja', 'jede',
|
||||
'jedem', 'jeden', 'jeder', 'jetzt', 'kann', 'kannst', 'kein', 'keine', 'können', 'könnt', 'man', 'mein',
|
||||
'meine', 'mich', 'mir', 'mit', 'muss', 'musst', 'nach', 'nicht', 'nichts', 'noch', 'nun', 'nur', 'ob', 'oder',
|
||||
'ohne', 'sehr', 'sein', 'seine', 'sich', 'sie', 'sind', 'so', 'soll', 'sollte', 'über', 'um', 'und', 'uns',
|
||||
'unser', 'unsere', 'unter', 'viel', 'vom', 'von', 'vor', 'war', 'waren', 'was', 'weil', 'welche', 'welchem',
|
||||
'welchen', 'welcher', 'welches', 'wenn', 'wer', 'werde', 'werden', 'wie', 'wieder', 'will', 'wir', 'wird',
|
||||
'wo', 'woher', 'wohin', 'zu', 'zum', 'zur', 'zwischen', 'bitte', 'gerne', 'euren', 'eurem', 'eurer',
|
||||
];
|
||||
|
||||
/**
|
||||
* The terms of a query: lower-cased, stopwords and short tokens dropped, unique.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public static function terms(string $text): array
|
||||
{
|
||||
$tokens = preg_split('/[^\p{L}\p{N}]+/u', mb_strtolower($text), -1, \PREG_SPLIT_NO_EMPTY) ?: [];
|
||||
$terms = [];
|
||||
|
||||
foreach ($tokens as $token) {
|
||||
if (mb_strlen($token) < self::MIN_TOKEN_LENGTH || in_array($token, self::STOPWORDS, true)) {
|
||||
continue;
|
||||
}
|
||||
$terms[$token] = true;
|
||||
}
|
||||
|
||||
return array_slice(array_keys($terms), 0, self::MAX_TERMS);
|
||||
}
|
||||
|
||||
/**
|
||||
* The boolean-mode query, or null when nothing searchable is left.
|
||||
*/
|
||||
public static function booleanQuery(string $text): ?string
|
||||
{
|
||||
$terms = self::terms($text);
|
||||
|
||||
return [] === $terms ? null : implode(' ', array_map(static fn (string $term): string => $term.'*', $terms));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot\Search;
|
||||
|
||||
/**
|
||||
* Cuts a window of about LENGTH characters out of a document, around its first match.
|
||||
*
|
||||
* The excerpt keeps search results small; get_document fetches the rest.
|
||||
*/
|
||||
final class Excerpt
|
||||
{
|
||||
public const LENGTH = 600;
|
||||
|
||||
/**
|
||||
* @param list<string> $terms lower-cased, matched as prefixes like the full-text search
|
||||
*/
|
||||
public static function around(string $text, array $terms, int $length = self::LENGTH): string
|
||||
{
|
||||
$text = trim((string) preg_replace('/\s+/u', ' ', $text));
|
||||
$textLength = mb_strlen($text);
|
||||
|
||||
if ($textLength <= $length) {
|
||||
return $text;
|
||||
}
|
||||
|
||||
$position = null;
|
||||
$lower = mb_strtolower($text);
|
||||
foreach ($terms as $term) {
|
||||
$found = mb_strpos($lower, $term);
|
||||
if (false !== $found && (null === $position || $found < $position)) {
|
||||
$position = $found;
|
||||
}
|
||||
}
|
||||
|
||||
// Start a third of the window before the match, so it is read in context.
|
||||
$start = max(0, min(($position ?? 0) - intdiv($length, 3), $textLength - $length));
|
||||
$excerpt = mb_substr($text, $start, $length);
|
||||
|
||||
if ($start > 0) {
|
||||
$space = mb_strpos($excerpt, ' ');
|
||||
$excerpt = '…'.(false !== $space && $space < 40 ? mb_substr($excerpt, $space + 1) : $excerpt);
|
||||
}
|
||||
|
||||
if ($start + $length < $textLength) {
|
||||
$space = mb_strrpos($excerpt, ' ');
|
||||
$excerpt = (false !== $space && mb_strlen($excerpt) - $space < 40 ? mb_substr($excerpt, 0, $space) : $excerpt).'…';
|
||||
}
|
||||
|
||||
return $excerpt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,653 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Chatbot;
|
||||
|
||||
use App\Chatbot\Search\ContentSearchQuery;
|
||||
use App\Entity\ContentDocument;
|
||||
use App\Model\Chatbot\TripCriteria;
|
||||
use App\Model\Export;
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
use App\Service\ProductExportBuilder;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
|
||||
/**
|
||||
* Trips as the chatbot sees them: TYPO3's editorial product joined to our dates, rooms,
|
||||
* prices and pickups from the product export.
|
||||
*
|
||||
* The join is on the product code; several TYPO3 products may share one, and each is its
|
||||
* own trip. Hotels are joined on any of the TYPO3 hotel's `codes`, because BusPro uses the
|
||||
* matchcode aliases; a code carried by more than one hotel resolves to the one whose own
|
||||
* `code` it is.
|
||||
*
|
||||
* A room is bookable unless its status is `Buchungsstop`. `available` counts rooms, so the
|
||||
* places of a date x hotel are the sum of `available x maxPax` over its bookable rooms.
|
||||
* Room counts never leave this class: the website does not show them, so the tools say
|
||||
* only whether a date is freely bookable or on request.
|
||||
*/
|
||||
class TripCatalog
|
||||
{
|
||||
/** The feed's only language for now; trips are joined in it. */
|
||||
public const LANGUAGE = 'de';
|
||||
|
||||
private const STATUS_BOOKING_STOP = 'Buchungsstop';
|
||||
private const STATUS_FREE = 'Frei';
|
||||
private const AVAILABILITY_FREE = 'frei';
|
||||
private const AVAILABILITY_ON_REQUEST = 'auf Anfrage';
|
||||
|
||||
private const NEXT_DATES = 3;
|
||||
private const DATES_WITH_PICKUPS = 3;
|
||||
private const BODY_LIMIT = 20000;
|
||||
private const HOTEL_BODY_LIMIT = 8000;
|
||||
|
||||
private const RECORD_TYPES = ['product', 'hotel', 'region', 'country', 'city', 'concept'];
|
||||
|
||||
public function __construct(
|
||||
private readonly ProductExportBuilder $exportBuilder,
|
||||
private readonly ContentDocumentRepository $repository,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{results: list<array<string, mixed>>, dataAsOf: ?string}
|
||||
*/
|
||||
public function search(TripCriteria $criteria): array
|
||||
{
|
||||
$catalog = $this->load();
|
||||
$products = $this->candidateProducts($catalog, $criteria);
|
||||
$today = $this->today();
|
||||
$results = [];
|
||||
|
||||
foreach ($products as $product) {
|
||||
$data = $product->getData() ?? [];
|
||||
$dates = [];
|
||||
$hotelCodes = [];
|
||||
|
||||
foreach ($catalog['rowsByCode'][$this->normalize($data['code'] ?? null)] ?? [] as $row) {
|
||||
$summary = $this->summarize($row);
|
||||
|
||||
if (!$this->qualifies($row, $summary, $criteria, $today)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$dateId = $row['date']['busProId'];
|
||||
$hotelCodes[$this->normalize($row['hotel']['code'])] = true;
|
||||
$current = $dates[$dateId] ?? null;
|
||||
|
||||
$dates[$dateId] = [
|
||||
'dateFrom' => $row['date']['from'],
|
||||
'dateTo' => $row['date']['to'],
|
||||
'priceFrom' => null === $current ? $summary['priceFrom'] : $this->minPrice($current['priceFrom'], $summary['priceFrom']),
|
||||
'availability' => self::AVAILABILITY_FREE === ($current['availability'] ?? null) ? self::AVAILABILITY_FREE : $summary['availability'],
|
||||
];
|
||||
}
|
||||
|
||||
if ([] === $dates) {
|
||||
continue;
|
||||
}
|
||||
|
||||
usort($dates, static fn (array $a, array $b): int => $a['dateFrom'] <=> $b['dateFrom']);
|
||||
|
||||
$results[] = $this->tripHeader($catalog, $product) + [
|
||||
'hotels' => $this->hotelNames($catalog, array_keys($hotelCodes)),
|
||||
'matchingDates' => count($dates),
|
||||
'nextDates' => array_slice($dates, 0, self::NEXT_DATES),
|
||||
];
|
||||
}
|
||||
|
||||
usort($results, static fn (array $a, array $b): int => [$a['nextDates'][0]['dateFrom'], $a['name']] <=> [$b['nextDates'][0]['dateFrom'], $b['name']]);
|
||||
|
||||
return ['results' => array_slice($results, 0, $criteria->limit), 'dataAsOf' => $this->dataAsOf($catalog['export'])];
|
||||
}
|
||||
|
||||
/**
|
||||
* Everything about one trip, or null for a code no published product carries.
|
||||
*
|
||||
* @return array<string, mixed>|null
|
||||
*/
|
||||
public function get(string $code, ?string $hotelCode = null): ?array
|
||||
{
|
||||
$catalog = $this->load();
|
||||
$products = $catalog['productsByCode'][$this->normalize($code)] ?? [];
|
||||
|
||||
if ([] === $products) {
|
||||
return null;
|
||||
}
|
||||
|
||||
usort($products, static fn (ContentDocument $a, ContentDocument $b): int => [null === $a->getUrl(), $a->getUid()] <=> [null === $b->getUrl(), $b->getUid()]);
|
||||
$product = $products[0];
|
||||
$today = $this->today();
|
||||
|
||||
$rows = array_values(array_filter(
|
||||
$catalog['rowsByCode'][$this->normalize($code)] ?? [],
|
||||
fn (array $row): bool => $row['date']['from'] >= $today,
|
||||
));
|
||||
|
||||
if (null !== $hotelCode && '' !== trim($hotelCode)) {
|
||||
$wanted = $this->hotelCodesFor($catalog, $hotelCode);
|
||||
$rows = array_values(array_filter($rows, fn (array $row): bool => in_array($this->normalize($row['hotel']['code']), $wanted, true)));
|
||||
}
|
||||
|
||||
usort($rows, static fn (array $a, array $b): int => [$a['date']['from'], $a['hotel']['code']] <=> [$b['date']['from'], $b['hotel']['code']]);
|
||||
|
||||
$hotelCodes = array_values(array_unique(array_map(fn (array $row): string => $this->normalize($row['hotel']['code']), $rows)));
|
||||
|
||||
return $this->tripHeader($catalog, $product) + [
|
||||
'alsoPublishedAt' => array_values(array_map(
|
||||
static fn (ContentDocument $other): array => ['site' => $other->getSite(), 'url' => $other->getUrl()],
|
||||
array_filter(array_slice($products, 1), static fn (ContentDocument $other): bool => null !== $other->getUrl()),
|
||||
)),
|
||||
'description' => $this->truncate($product->getBodyMd(), self::BODY_LIMIT),
|
||||
'hotels' => $this->hotelDetails($catalog, $hotelCodes, $product),
|
||||
'dates' => $this->dates($catalog, $rows),
|
||||
'dataAsOf' => $this->dataAsOf($catalog['export']),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs the codes the join cannot place, on both sides. The same gaps the website's
|
||||
* product import tolerates, worth a periodic look.
|
||||
*/
|
||||
public function logUnmatchedCodes(): void
|
||||
{
|
||||
$catalog = $this->load();
|
||||
|
||||
$exportProducts = array_keys($catalog['rowsByCode']);
|
||||
$contentProducts = array_keys($catalog['productsByCode']);
|
||||
|
||||
$exportHotels = [];
|
||||
foreach ($catalog['export']->rows as $row) {
|
||||
$exportHotels[$this->normalize($row['hotel']['code'] ?? null)] = true;
|
||||
}
|
||||
unset($exportHotels['']);
|
||||
|
||||
$ambiguous = [];
|
||||
foreach ($catalog['hotelsByCode'] as $hotelCode => $hotels) {
|
||||
if (count($hotels) > 1) {
|
||||
$ambiguous[$hotelCode] = array_map(static fn (ContentDocument $hotel): int => $hotel->getUid(), $hotels);
|
||||
}
|
||||
}
|
||||
|
||||
$context = [
|
||||
'exportProductsWithoutContent' => array_values(array_diff($exportProducts, $contentProducts)),
|
||||
'contentProductsWithoutDates' => array_values(array_diff($contentProducts, $exportProducts)),
|
||||
'exportHotelsWithoutContent' => array_values(array_diff(array_keys($exportHotels), array_keys($catalog['hotelsByCode']))),
|
||||
'ambiguousHotelCodes' => $ambiguous,
|
||||
];
|
||||
|
||||
$this->logger->info('Content feed: unmatched codes', $context + [
|
||||
'counts' => array_map('count', $context),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{
|
||||
* export: Export,
|
||||
* records: array<string, ContentDocument>,
|
||||
* productsByCode: array<string, list<ContentDocument>>,
|
||||
* hotelsByCode: array<string, list<ContentDocument>>,
|
||||
* rowsByCode: array<string, list<array<string, mixed>>>,
|
||||
* }
|
||||
*/
|
||||
private function load(): array
|
||||
{
|
||||
$export = $this->exportBuilder->build();
|
||||
$records = $this->repository->findRecords(self::RECORD_TYPES, self::LANGUAGE);
|
||||
|
||||
$productsByCode = [];
|
||||
$hotelsByCode = [];
|
||||
|
||||
foreach ($records as $record) {
|
||||
$data = $record->getData() ?? [];
|
||||
|
||||
if ('product' === $record->getType() && '' !== ($code = $this->normalize($data['code'] ?? null))) {
|
||||
$productsByCode[$code][] = $record;
|
||||
}
|
||||
|
||||
if ('hotel' === $record->getType()) {
|
||||
$codes = is_array($data['codes'] ?? null) ? $data['codes'] : [];
|
||||
$codes[] = $data['code'] ?? null;
|
||||
|
||||
foreach (array_unique(array_map($this->normalize(...), $codes)) as $code) {
|
||||
if ('' !== $code) {
|
||||
$hotelsByCode[$code][] = $record;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A code that is one hotel's own and another's alias resolves to the owner.
|
||||
foreach ($hotelsByCode as $code => $hotels) {
|
||||
usort($hotels, fn (ContentDocument $a, ContentDocument $b): int => [$this->normalize($a->getData()['code'] ?? null) !== $code, $a->getUid()] <=> [$this->normalize($b->getData()['code'] ?? null) !== $code, $b->getUid()]);
|
||||
$hotelsByCode[$code] = $hotels;
|
||||
}
|
||||
|
||||
$rowsByCode = [];
|
||||
foreach ($export->rows as $row) {
|
||||
$code = $this->normalize($row['product']['code'] ?? null);
|
||||
if ('' !== $code) {
|
||||
$rowsByCode[$code][] = $row;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'export' => $export,
|
||||
'records' => $records,
|
||||
'productsByCode' => $productsByCode,
|
||||
'hotelsByCode' => $hotelsByCode,
|
||||
'rowsByCode' => $rowsByCode,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Published products narrowed by everything the content index can resolve: site, day
|
||||
* trip, concept, destination and free text.
|
||||
*
|
||||
* @param array<string, mixed> $catalog
|
||||
*
|
||||
* @return list<ContentDocument>
|
||||
*/
|
||||
private function candidateProducts(array $catalog, TripCriteria $criteria): array
|
||||
{
|
||||
/** @var array<string, ContentDocument> $records */
|
||||
$records = $catalog['records'];
|
||||
|
||||
$conceptUids = null === $criteria->concept ? null : $this->matchingUids($records, ['concept'], $criteria->concept, matchCode: true);
|
||||
$countryUids = null === $criteria->country ? null : $this->matchingUids($records, ['country'], $criteria->country, matchCode: true);
|
||||
$regionUids = null === $criteria->region ? null : $this->matchingUids($records, ['region'], $criteria->region);
|
||||
$cityUids = null === $criteria->region ? null : $this->matchingUids($records, ['city'], $criteria->region);
|
||||
$queryUids = null === $criteria->query ? null : $this->productUidsMatchingText($records, $criteria->query);
|
||||
|
||||
$products = [];
|
||||
|
||||
foreach ($records as $record) {
|
||||
if ('product' !== $record->getType() || null === $record->getUrl()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$data = $record->getData() ?? [];
|
||||
|
||||
// Closed group trips and staff trainings are not offered; get() still answers
|
||||
// someone who names their code.
|
||||
if (false === ($data['bookable'] ?? null)
|
||||
|| $record->isHiddenFromSearch()
|
||||
|| (null !== $criteria->site && $record->getSite() !== $criteria->site)
|
||||
|| (null !== $criteria->daytrip && (bool) ($data['daytrip'] ?? false) !== $criteria->daytrip)
|
||||
|| (null !== $conceptUids && !in_array($data['conceptUid'] ?? null, $conceptUids, true))
|
||||
|| (null !== $countryUids && !in_array($data['countryUid'] ?? null, $countryUids, true))
|
||||
|| (null !== $queryUids && !in_array($record->getUid(), $queryUids, true))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (null !== $regionUids) {
|
||||
$productRegions = array_merge([$data['regionUid'] ?? null], is_array($data['additionalRegionUids'] ?? null) ? $data['additionalRegionUids'] : []);
|
||||
|
||||
if ([] === array_intersect($productRegions, $regionUids) && !in_array($data['cityUid'] ?? null, $cityUids ?? [], true)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$products[] = $record;
|
||||
}
|
||||
|
||||
return $products;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uids of the records of the given types whose name contains the text, or whose code
|
||||
* equals it.
|
||||
*
|
||||
* @param array<string, ContentDocument> $records
|
||||
* @param list<string> $types
|
||||
*
|
||||
* @return list<int>
|
||||
*/
|
||||
private function matchingUids(array $records, array $types, string $text, bool $matchCode = false): array
|
||||
{
|
||||
$needle = mb_strtolower(trim($text));
|
||||
$uids = [];
|
||||
|
||||
foreach ($records as $record) {
|
||||
if (!in_array($record->getType(), $types, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$name = mb_strtolower($record->getTitle());
|
||||
$code = $this->normalize($record->getData()['code'] ?? null);
|
||||
|
||||
if (('' !== $name && (str_contains($name, $needle) || str_contains($needle, $name)))
|
||||
|| ($matchCode && '' !== $code && $code === $this->normalize($text))) {
|
||||
$uids[] = $record->getUid();
|
||||
}
|
||||
}
|
||||
|
||||
return $uids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Products reached by a full-text match on themselves or on a hotel, destination or
|
||||
* concept they belong to.
|
||||
*
|
||||
* @param array<string, ContentDocument> $records
|
||||
*
|
||||
* @return list<int>
|
||||
*/
|
||||
private function productUidsMatchingText(array $records, string $text): array
|
||||
{
|
||||
$booleanQuery = ContentSearchQuery::booleanQuery($text);
|
||||
|
||||
if (null === $booleanQuery) {
|
||||
return array_values(array_map(static fn (ContentDocument $record): int => $record->getUid(), array_filter($records, static fn (ContentDocument $record): bool => 'product' === $record->getType())));
|
||||
}
|
||||
|
||||
$matched = [];
|
||||
foreach ($this->repository->matchRecords($booleanQuery, self::RECORD_TYPES, self::LANGUAGE) as $hit) {
|
||||
$matched[$hit['type']][$hit['uid']] = true;
|
||||
}
|
||||
|
||||
$uids = [];
|
||||
|
||||
foreach ($records as $record) {
|
||||
if ('product' !== $record->getType()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$data = $record->getData() ?? [];
|
||||
$hotelUids = is_array($data['hotelUids'] ?? null) ? $data['hotelUids'] : [];
|
||||
$regionUids = array_merge([$data['regionUid'] ?? null], is_array($data['additionalRegionUids'] ?? null) ? $data['additionalRegionUids'] : []);
|
||||
|
||||
$hit = isset($matched['product'][$record->getUid()])
|
||||
|| [] !== array_intersect($hotelUids, array_keys($matched['hotel'] ?? []))
|
||||
|| [] !== array_intersect($regionUids, array_keys($matched['region'] ?? []))
|
||||
|| isset($matched['country'][$data['countryUid'] ?? -1])
|
||||
|| isset($matched['city'][$data['cityUid'] ?? -1])
|
||||
|| isset($matched['concept'][$data['conceptUid'] ?? -1]);
|
||||
|
||||
// Hotels also name their products, for products whose hotelUids miss them.
|
||||
if (!$hit) {
|
||||
foreach (array_keys($matched['hotel'] ?? []) as $hotelUid) {
|
||||
$hotelProducts = ($records['hotel:'.$hotelUid] ?? null)?->getData()['productUids'] ?? [];
|
||||
if (is_array($hotelProducts) && in_array($record->getUid(), $hotelProducts, true)) {
|
||||
$hit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($hit) {
|
||||
$uids[] = $record->getUid();
|
||||
}
|
||||
}
|
||||
|
||||
return $uids;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $row
|
||||
*
|
||||
* @return array{priceFrom: ?float, places: int, availability: ?string}
|
||||
*/
|
||||
private function summarize(array $row): array
|
||||
{
|
||||
$priceFrom = null;
|
||||
$places = 0;
|
||||
$availability = null;
|
||||
|
||||
foreach ($row['rooms'] ?? [] as $room) {
|
||||
if (self::STATUS_BOOKING_STOP === ($room['status'] ?? null)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$places += (int) ($room['available'] ?? 0) * (int) ($room['maxPax'] ?? 0);
|
||||
$availability = self::STATUS_FREE === ($room['status'] ?? null) || self::AVAILABILITY_FREE === $availability
|
||||
? self::AVAILABILITY_FREE
|
||||
: self::AVAILABILITY_ON_REQUEST;
|
||||
|
||||
if (is_int($room['price'] ?? null) || is_float($room['price'] ?? null)) {
|
||||
$priceFrom = $this->minPrice($priceFrom, (float) $room['price']);
|
||||
}
|
||||
}
|
||||
|
||||
return ['priceFrom' => $priceFrom, 'places' => $places, 'availability' => $availability];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $row
|
||||
* @param array{priceFrom: ?float, places: int, availability: ?string} $summary
|
||||
*/
|
||||
private function qualifies(array $row, array $summary, TripCriteria $criteria, string $today): bool
|
||||
{
|
||||
$from = $row['date']['from'];
|
||||
$to = $row['date']['to'];
|
||||
|
||||
return $from >= $today
|
||||
&& null !== $summary['availability']
|
||||
&& (null === $criteria->dateFrom || $to >= $criteria->dateFrom->format('Y-m-d'))
|
||||
&& (null === $criteria->dateTo || $from <= $criteria->dateTo->format('Y-m-d'))
|
||||
&& (null === $criteria->maxPrice || (null !== $summary['priceFrom'] && $summary['priceFrom'] <= $criteria->maxPrice))
|
||||
&& (null === $criteria->persons || $summary['places'] >= $criteria->persons);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $catalog
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function tripHeader(array $catalog, ContentDocument $product): array
|
||||
{
|
||||
$data = $product->getData() ?? [];
|
||||
$name = fn (string $type, mixed $uid): ?string => is_int($uid) ? ($catalog['records'][$type.':'.$uid] ?? null)?->getTitle() : null;
|
||||
|
||||
return [
|
||||
'code' => $data['code'] ?? null,
|
||||
'name' => $product->getTitle(),
|
||||
'url' => $product->getUrl(),
|
||||
'site' => $product->getSite(),
|
||||
'destination' => array_filter([
|
||||
'country' => $name('country', $data['countryUid'] ?? null),
|
||||
'region' => $name('region', $data['regionUid'] ?? null),
|
||||
'city' => $name('city', $data['cityUid'] ?? null),
|
||||
]),
|
||||
'concept' => $name('concept', $data['conceptUid'] ?? null),
|
||||
'daytrip' => (bool) ($data['daytrip'] ?? false),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $catalog
|
||||
* @param list<string> $hotelCodes normalized
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function hotelNames(array $catalog, array $hotelCodes): array
|
||||
{
|
||||
$names = [];
|
||||
foreach ($hotelCodes as $code) {
|
||||
$hotel = $catalog['hotelsByCode'][$code][0] ?? null;
|
||||
if (null !== $hotel && '' !== $hotel->getTitle()) {
|
||||
$names[$hotel->getTitle()] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return array_keys($names);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $catalog
|
||||
* @param list<string> $hotelCodes normalized codes of the hotels with dates
|
||||
*
|
||||
* @return list<array<string, mixed>>
|
||||
*/
|
||||
private function hotelDetails(array $catalog, array $hotelCodes, ContentDocument $product): array
|
||||
{
|
||||
$hotels = [];
|
||||
|
||||
foreach ($hotelCodes as $code) {
|
||||
$hotel = $catalog['hotelsByCode'][$code][0] ?? null;
|
||||
if (null !== $hotel) {
|
||||
$hotels[$hotel->getUid()] = $hotel;
|
||||
}
|
||||
}
|
||||
|
||||
// A trip without upcoming dates still describes its hotels.
|
||||
if ([] === $hotels) {
|
||||
foreach ((array) ($product->getData()['hotelUids'] ?? []) as $uid) {
|
||||
$hotel = is_int($uid) ? ($catalog['records']['hotel:'.$uid] ?? null) : null;
|
||||
if (null !== $hotel) {
|
||||
$hotels[$hotel->getUid()] = $hotel;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return array_values(array_map(fn (ContentDocument $hotel): array => [
|
||||
'code' => $hotel->getData()['code'] ?? null,
|
||||
'name' => $hotel->getTitle(),
|
||||
'url' => $hotel->getUrl(),
|
||||
'facts' => is_array($hotel->getData()['facts'] ?? null) ? array_filter($hotel->getData()['facts'], static fn (mixed $value): bool => null !== $value && '' !== $value) : [],
|
||||
'description' => $this->truncate($hotel->getBodyMd(), self::HOTEL_BODY_LIMIT),
|
||||
], $hotels));
|
||||
}
|
||||
|
||||
/**
|
||||
* Upcoming dates with their bookable rooms per hotel. Pickups for the first few only:
|
||||
* they rarely differ between dates and would otherwise dominate the answer.
|
||||
*
|
||||
* @param array<string, mixed> $catalog
|
||||
* @param list<array<string, mixed>> $rows sorted by date
|
||||
*
|
||||
* @return list<array<string, mixed>>
|
||||
*/
|
||||
private function dates(array $catalog, array $rows): array
|
||||
{
|
||||
$dates = [];
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$dateId = $row['date']['busProId'];
|
||||
$hotel = $catalog['hotelsByCode'][$this->normalize($row['hotel']['code'])][0] ?? null;
|
||||
|
||||
$rooms = [];
|
||||
foreach ($row['rooms'] ?? [] as $room) {
|
||||
if (self::STATUS_BOOKING_STOP === ($room['status'] ?? null)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rooms[] = array_filter([
|
||||
'room' => $room['label'] ?? null,
|
||||
'board' => $room['boardCode'] ?? null,
|
||||
'nights' => $room['nights'] ?? null,
|
||||
'price' => $room['price'] ?? null,
|
||||
'availability' => self::STATUS_FREE === ($room['status'] ?? null) ? self::AVAILABILITY_FREE : self::AVAILABILITY_ON_REQUEST,
|
||||
], static fn (mixed $value): bool => null !== $value);
|
||||
}
|
||||
|
||||
$dates[$dateId] ??= [
|
||||
'dateFrom' => $row['date']['from'],
|
||||
'dateTo' => $row['date']['to'],
|
||||
'hotels' => [],
|
||||
'pickups' => $this->pickups($row),
|
||||
];
|
||||
|
||||
$dates[$dateId]['hotels'][] = [
|
||||
'hotel' => $hotel?->getTitle() ?? $row['hotel']['code'],
|
||||
'hotelCode' => $row['hotel']['code'],
|
||||
'rooms' => $rooms,
|
||||
'bookable' => [] !== $rooms,
|
||||
];
|
||||
}
|
||||
|
||||
$dates = array_values($dates);
|
||||
|
||||
foreach ($dates as $i => $date) {
|
||||
if ($i >= self::DATES_WITH_PICKUPS) {
|
||||
unset($dates[$i]['pickups']);
|
||||
}
|
||||
}
|
||||
|
||||
return $dates;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $row
|
||||
*
|
||||
* @return list<array<string, mixed>>
|
||||
*/
|
||||
private function pickups(array $row): array
|
||||
{
|
||||
$pickups = [];
|
||||
|
||||
foreach ($row['pickups'] ?? [] as $pickup) {
|
||||
if (0 === ($pickup['busProId'] ?? 0) || null === ($pickup['city'] ?? null)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$pickups[] = array_filter([
|
||||
'city' => $pickup['city'],
|
||||
'place' => $pickup['street'] ?? null,
|
||||
// A clock reading without an offset, as BusPro states it.
|
||||
'departure' => is_string($pickup['datetime'] ?? null) ? str_replace('T', ' ', substr($pickup['datetime'], 0, 16)) : null,
|
||||
'surcharge' => $pickup['price'] ?? null,
|
||||
], static fn (mixed $value): bool => null !== $value);
|
||||
}
|
||||
|
||||
return $pickups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalized codes of the hotel a code names: all its own codes when TYPO3 knows the
|
||||
* hotel, else just the code.
|
||||
*
|
||||
* @param array<string, mixed> $catalog
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function hotelCodesFor(array $catalog, string $hotelCode): array
|
||||
{
|
||||
$hotel = $catalog['hotelsByCode'][$this->normalize($hotelCode)][0] ?? null;
|
||||
|
||||
if (null === $hotel) {
|
||||
return [$this->normalize($hotelCode)];
|
||||
}
|
||||
|
||||
$codes = is_array($hotel->getData()['codes'] ?? null) ? $hotel->getData()['codes'] : [];
|
||||
$codes[] = $hotel->getData()['code'] ?? null;
|
||||
|
||||
return array_values(array_filter(array_unique(array_map($this->normalize(...), $codes))));
|
||||
}
|
||||
|
||||
private function dataAsOf(Export $export): string
|
||||
{
|
||||
return ($export->sourceGeneratedAt ?? $export->generatedAt)->format('c');
|
||||
}
|
||||
|
||||
private function truncate(string $text, int $limit): string
|
||||
{
|
||||
if (mb_strlen($text) <= $limit) {
|
||||
return $text;
|
||||
}
|
||||
|
||||
return mb_substr($text, 0, $limit)."\n\n[… gekürzt, vollständiger Text unter der URL]";
|
||||
}
|
||||
|
||||
private function minPrice(?float $a, ?float $b): ?float
|
||||
{
|
||||
if (null === $a || null === $b) {
|
||||
return $a ?? $b;
|
||||
}
|
||||
|
||||
return min($a, $b);
|
||||
}
|
||||
|
||||
private function today(): string
|
||||
{
|
||||
return $this->clock->now()->setTimezone(new \DateTimeZone('Europe/Berlin'))->format('Y-m-d');
|
||||
}
|
||||
|
||||
private function normalize(mixed $code): string
|
||||
{
|
||||
return is_string($code) ? strtoupper(trim($code)) : '';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\Repository\ContentSyncStateRepository;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
#[AsCommand(
|
||||
name: 'app:content:check-freshness',
|
||||
description: 'Fails when the last accepted content manifest is too old.',
|
||||
)]
|
||||
/**
|
||||
* The health check of the content sync. A stalled sync otherwise shows only as a chatbot
|
||||
* giving stale answers. Scheduled, a failure reaches the schedule's email_on_failure.
|
||||
*/
|
||||
class ContentCheckFreshnessCommand extends Command
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ContentSyncStateRepository $stateRepository,
|
||||
private readonly ClockInterface $clock,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this->addOption('max-age', null, InputOption::VALUE_REQUIRED, 'Maximum age of the last good manifest, in minutes', '60');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
$state = $this->stateRepository->get();
|
||||
$generatedAt = $state->getManifestGeneratedAt();
|
||||
$maxAge = (int) $input->getOption('max-age');
|
||||
|
||||
if (null === $generatedAt) {
|
||||
$io->error('No content manifest has been accepted yet.');
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$ageMinutes = (int) floor(($this->clock->now()->getTimestamp() - $generatedAt->getTimestamp()) / 60);
|
||||
|
||||
if ($ageMinutes > $maxAge) {
|
||||
$io->error(sprintf('The last good content manifest is %d minutes old (generated %s). Last run error: %s', $ageMinutes, $generatedAt->format('c'), $state->getLastError() ?? 'none'));
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$io->success(sprintf('The last good content manifest is %d minutes old.', $ageMinutes));
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\ContentFeed\ContentSyncer;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Command\LockableTrait;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
#[AsCommand(
|
||||
name: 'app:content:sync',
|
||||
description: 'Mirrors the TYPO3 content feed for the chatbot MCP server.',
|
||||
)]
|
||||
/**
|
||||
* Console entrypoint for the content feed sync.
|
||||
*
|
||||
* Meant to run every 15 minutes, plus once a week with --refresh in the night window.
|
||||
* Outside 01:00-05:00 Europe/Berlin a run fetches at most 50 pages; --ignore-window lifts
|
||||
* that for a deliberate manual run, such as the first sync on staging.
|
||||
*/
|
||||
class ContentSyncCommand extends Command
|
||||
{
|
||||
use LockableTrait;
|
||||
|
||||
public function __construct(private readonly ContentSyncer $syncer)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this
|
||||
->addOption('refresh', null, InputOption::VALUE_NONE, sprintf('Also refetch items stored more than %d days ago, oldest first', ContentSyncer::REFRESH_AFTER_DAYS))
|
||||
->addOption('ignore-window', null, InputOption::VALUE_NONE, sprintf('Fetch every due page, not just %d, outside the night window', ContentSyncer::PAGES_OUTSIDE_WINDOW))
|
||||
;
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
|
||||
if (!$this->lock()) {
|
||||
$io->warning('Another content sync is running.');
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
try {
|
||||
$report = $this->syncer->run((bool) $input->getOption('refresh'), (bool) $input->getOption('ignore-window'));
|
||||
} finally {
|
||||
$this->release();
|
||||
}
|
||||
|
||||
$io->definitionList(
|
||||
['Added' => $report->added],
|
||||
['Deleted' => $report->deleted],
|
||||
['Records fetched' => $report->records],
|
||||
['Pages fetched' => $report->pages],
|
||||
['Failed' => $report->failed],
|
||||
['Pages deferred' => $report->deferred],
|
||||
);
|
||||
|
||||
if (null !== $report->pagePhaseEndedBy) {
|
||||
$io->warning('Page phase ended early: '.$report->pagePhaseEndedBy);
|
||||
}
|
||||
|
||||
if (null !== $report->stoppedBy) {
|
||||
$io->error('Run stopped: '.$report->stoppedBy);
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$io->success('Content feed synced.');
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use App\ContentFeed\Exception\ContentFeedStoppedException;
|
||||
use App\ContentFeed\Exception\ContentFeedUnavailableException;
|
||||
use App\ContentFeed\Exception\ContentRequestFailedException;
|
||||
use App\ContentFeed\Exception\PagePhaseEndedException;
|
||||
use App\Model\ContentFeed\ContentManifest;
|
||||
use App\Model\ContentFeed\ManifestItem;
|
||||
use App\Model\ContentFeed\PageFetchResult;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
|
||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||
use Symfony\Contracts\HttpClient\ResponseInterface;
|
||||
|
||||
/**
|
||||
* Reads the TYPO3 content feed: the manifest, records by uid, and rendered page fragments.
|
||||
*
|
||||
* The website is an old TYPO3 install on which every page fetch is a full frontend render,
|
||||
* so the load rules here are a contract with its maintainers, not tuning:
|
||||
*
|
||||
* - one request at a time, never parallel;
|
||||
* - at most one page fetch per second;
|
||||
* - after a response slower than 2 s, wait 5 s; after three in a row, no more pages this run;
|
||||
* - a 503 (the kill switch) or three consecutive 5xx responses or timeouts end the run.
|
||||
*
|
||||
* One instance serves one run: call startRun() first, it resets the counters.
|
||||
*/
|
||||
class ContentFeedClient
|
||||
{
|
||||
public const RECORDS_BATCH_SIZE = 100;
|
||||
|
||||
private const TIMEOUT_SECONDS = 10.0;
|
||||
private const SLOW_SECONDS = 2.0;
|
||||
private const SLOW_PAUSE_SECONDS = 5.0;
|
||||
private const PAGE_INTERVAL_SECONDS = 1.0;
|
||||
private const MAX_CONSECUTIVE_FAILURES = 3;
|
||||
private const MAX_CONSECUTIVE_SLOW = 3;
|
||||
|
||||
private int $consecutiveFailures = 0;
|
||||
private int $consecutiveSlow = 0;
|
||||
private ?\DateTimeImmutable $notBefore = null;
|
||||
private ?\DateTimeImmutable $lastPageAt = null;
|
||||
|
||||
/** @var list<string> */
|
||||
private array $siteHosts = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly HttpClientInterface $httpClient,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly string $baseUrl,
|
||||
private readonly string $apiKey,
|
||||
) {
|
||||
}
|
||||
|
||||
public function startRun(): void
|
||||
{
|
||||
$this->consecutiveFailures = 0;
|
||||
$this->consecutiveSlow = 0;
|
||||
$this->notBefore = null;
|
||||
$this->lastPageAt = null;
|
||||
$this->siteHosts = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole manifest. Anything but a usable one stops the run: an empty TYPO3 is a bug,
|
||||
* not a deletion.
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function manifest(): ContentManifest
|
||||
{
|
||||
if ('' === $this->apiKey) {
|
||||
throw new ContentFeedStoppedException('APP_CMS_API_KEY is not configured');
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/manifest'), []);
|
||||
|
||||
if (null === $response || 200 !== $response['status']) {
|
||||
throw new ContentFeedStoppedException(sprintf('Manifest request failed (%s)', $response['status'] ?? 'timeout'));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentFeedStoppedException('Manifest is not valid JSON', 0, $e);
|
||||
}
|
||||
|
||||
if (!is_array($data) || !is_string($data['generatedAt'] ?? null) || !is_array($data['items'] ?? null) || [] === $data['items']) {
|
||||
throw new ContentFeedStoppedException('Manifest has no generatedAt or no items');
|
||||
}
|
||||
|
||||
$generatedAt = ManifestItem::localTime($data['generatedAt']);
|
||||
if (null === $generatedAt) {
|
||||
throw new ContentFeedStoppedException('Manifest generatedAt is unparseable');
|
||||
}
|
||||
|
||||
$items = [];
|
||||
foreach ($data['items'] as $raw) {
|
||||
try {
|
||||
$item = ManifestItem::fromArray(is_array($raw) ? $raw : []);
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
// A malformed entry would otherwise read as a deletion of whatever it was.
|
||||
throw new ContentFeedStoppedException('Manifest contains an unusable item: '.$e->getMessage(), 0, $e);
|
||||
}
|
||||
$items[$item->key()] = $item;
|
||||
}
|
||||
|
||||
$hosts = [];
|
||||
foreach (is_array($data['sites'] ?? null) ? $data['sites'] : [] as $site) {
|
||||
$host = is_array($site) && is_string($site['base'] ?? null) ? parse_url($site['base'], \PHP_URL_HOST) : null;
|
||||
if (is_string($host)) {
|
||||
$hosts[] = strtolower($host);
|
||||
}
|
||||
}
|
||||
$this->siteHosts = array_values(array_unique($hosts));
|
||||
|
||||
return new ContentManifest($generatedAt, $items, $this->siteHosts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records of one type by uid, keyed by uid. A requested uid that is absent is no
|
||||
* longer eligible.
|
||||
*
|
||||
* @param list<int> $uids at most RECORDS_BATCH_SIZE
|
||||
*
|
||||
* @return array<int, array<string, mixed>>
|
||||
*
|
||||
* @throws ContentRequestFailedException this batch failed, the run goes on
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function records(string $type, array $uids): array
|
||||
{
|
||||
if (count($uids) > self::RECORDS_BATCH_SIZE) {
|
||||
throw new \InvalidArgumentException(sprintf('At most %d uids per records request', self::RECORDS_BATCH_SIZE));
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/records'), ['type' => $type, 'uids' => implode(',', $uids)]);
|
||||
|
||||
if (null === $response) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s timed out', $type));
|
||||
}
|
||||
|
||||
if (200 !== $response['status']) {
|
||||
if ($response['status'] >= 500) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s failed (%d)', $type, $response['status']));
|
||||
}
|
||||
|
||||
// 403 and the like: misconfiguration, which the next request would hit too.
|
||||
throw new ContentFeedStoppedException(sprintf('Records request for %s answered %d', $type, $response['status']));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s is not valid JSON', $type), 0, $e);
|
||||
}
|
||||
|
||||
// Anything but the documented envelope fails the batch: read as an empty list, it
|
||||
// would delete every record requested.
|
||||
if (!is_array($data) || ($data['type'] ?? null) !== $type || !is_array($data['items'] ?? null) || !array_is_list($data['items'])) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s has no items list', $type));
|
||||
}
|
||||
|
||||
$records = [];
|
||||
foreach ($data['items'] as $record) {
|
||||
if (is_array($record) && is_int($record['uid'] ?? null)) {
|
||||
$records[$record['uid']] = $record;
|
||||
}
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
/**
|
||||
* The bare HTML fragment of one page, fetched at most once a second.
|
||||
*
|
||||
* @throws PagePhaseEndedException
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function page(string $contentUrl): PageFetchResult
|
||||
{
|
||||
$host = parse_url($contentUrl, \PHP_URL_HOST);
|
||||
|
||||
// The key goes with every request, so it only ever goes to a host of the feed's own sites.
|
||||
if (!is_string($host) || !in_array(strtolower($host), $this->siteHosts, true)) {
|
||||
$this->logger->warning('Content feed: contentUrl outside the manifest sites, skipped', ['contentUrl' => $contentUrl]);
|
||||
|
||||
return PageFetchResult::failed(0);
|
||||
}
|
||||
|
||||
if (null !== $this->lastPageAt) {
|
||||
$this->waitUntil($this->lastPageAt->modify(sprintf('+%d ms', (int) (self::PAGE_INTERVAL_SECONDS * 1000))));
|
||||
}
|
||||
$this->lastPageAt = $this->clock->now();
|
||||
|
||||
$response = $this->send($contentUrl, [], countSlow: true);
|
||||
|
||||
if (null === $response) {
|
||||
return PageFetchResult::transient();
|
||||
}
|
||||
|
||||
if (200 === $response['status']) {
|
||||
return PageFetchResult::ok($response['body']);
|
||||
}
|
||||
|
||||
return PageFetchResult::failed($response['status']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends one GET under the load rules. Null on a timeout or transport error.
|
||||
*
|
||||
* @param array<string, string> $query
|
||||
*
|
||||
* @return array{status: int, body: string}|null
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
* @throws PagePhaseEndedException
|
||||
*/
|
||||
private function send(string $url, array $query, bool $countSlow = false): ?array
|
||||
{
|
||||
if (null !== $this->notBefore) {
|
||||
$this->waitUntil($this->notBefore);
|
||||
$this->notBefore = null;
|
||||
}
|
||||
|
||||
$result = null;
|
||||
$duration = 0.0;
|
||||
|
||||
try {
|
||||
$response = $this->httpClient->request('GET', $url, [
|
||||
'query' => $query,
|
||||
'headers' => ['X-Api-Key' => $this->apiKey],
|
||||
'timeout' => self::TIMEOUT_SECONDS,
|
||||
'max_duration' => self::TIMEOUT_SECONDS,
|
||||
]);
|
||||
$result = ['status' => $response->getStatusCode(), 'body' => $response->getContent(false)];
|
||||
$duration = $this->duration($response);
|
||||
} catch (TransportExceptionInterface $e) {
|
||||
$this->logger->warning('Content feed: request failed', ['url' => $url, 'error' => $e->getMessage()]);
|
||||
$duration = self::TIMEOUT_SECONDS;
|
||||
}
|
||||
|
||||
if (null !== $result && 503 === $result['status']) {
|
||||
throw new ContentFeedUnavailableException($this->retryAfter($response));
|
||||
}
|
||||
|
||||
if (null === $result || $result['status'] >= 500) {
|
||||
if (++$this->consecutiveFailures >= self::MAX_CONSECUTIVE_FAILURES) {
|
||||
throw new ContentFeedStoppedException(sprintf('%d consecutive 5xx responses or timeouts', self::MAX_CONSECUTIVE_FAILURES));
|
||||
}
|
||||
} else {
|
||||
$this->consecutiveFailures = 0;
|
||||
}
|
||||
|
||||
if ($duration > self::SLOW_SECONDS) {
|
||||
$this->notBefore = $this->clock->now()->modify(sprintf('+%d ms', (int) (self::SLOW_PAUSE_SECONDS * 1000)));
|
||||
|
||||
if ($countSlow && ++$this->consecutiveSlow >= self::MAX_CONSECUTIVE_SLOW) {
|
||||
$this->consecutiveSlow = 0;
|
||||
throw new PagePhaseEndedException(sprintf('%d consecutive responses slower than %d s', self::MAX_CONSECUTIVE_SLOW, (int) self::SLOW_SECONDS));
|
||||
}
|
||||
} elseif ($countSlow) {
|
||||
$this->consecutiveSlow = 0;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function duration(ResponseInterface $response): float
|
||||
{
|
||||
$total = $response->getInfo('total_time');
|
||||
|
||||
return is_float($total) || is_int($total) ? (float) $total : 0.0;
|
||||
}
|
||||
|
||||
private function retryAfter(ResponseInterface $response): ?\DateTimeImmutable
|
||||
{
|
||||
$value = $response->getHeaders(false)['retry-after'][0] ?? null;
|
||||
|
||||
if (null === $value || '' === trim($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (ctype_digit(trim($value))) {
|
||||
return $this->clock->now()->modify(sprintf('+%d seconds', (int) trim($value)));
|
||||
}
|
||||
|
||||
$date = date_create_immutable($value);
|
||||
|
||||
return false === $date ? null : $date;
|
||||
}
|
||||
|
||||
private function waitUntil(\DateTimeImmutable $moment): void
|
||||
{
|
||||
$seconds = (float) $moment->format('U.u') - (float) $this->clock->now()->format('U.u');
|
||||
|
||||
if ($seconds > 0) {
|
||||
$this->clock->sleep($seconds);
|
||||
}
|
||||
}
|
||||
|
||||
private function apiUrl(string $path): string
|
||||
{
|
||||
return rtrim($this->baseUrl, '/').'/'.$path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use App\Chatbot\TripCatalog;
|
||||
use App\ContentFeed\Exception\ContentFeedStoppedException;
|
||||
use App\ContentFeed\Exception\ContentFeedUnavailableException;
|
||||
use App\ContentFeed\Exception\ContentRequestFailedException;
|
||||
use App\ContentFeed\Exception\PagePhaseEndedException;
|
||||
use App\Entity\ContentDocument;
|
||||
use App\Enum\ContentFetchState;
|
||||
use App\Model\ContentFeed\ContentManifest;
|
||||
use App\Model\ContentFeed\ContentSyncReport;
|
||||
use App\Model\ContentFeed\ManifestItem;
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
use App\Repository\ContentSyncStateRepository;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
|
||||
/**
|
||||
* Mirrors the TYPO3 content feed into content_document.
|
||||
*
|
||||
* A run reads the manifest, removes what it no longer lists, refetches the records and
|
||||
* pages whose `modified` changed, and re-merges detail pages into their records. A failed
|
||||
* manifest changes nothing. Bulk page work belongs in the night window: outside it, a run
|
||||
* fetches at most PAGES_OUTSIDE_WINDOW pages and leaves the rest for the next one.
|
||||
*
|
||||
* With `refresh`, items whose stored copy is older than REFRESH_AFTER_DAYS are refetched
|
||||
* regardless of `modified`, oldest first: the backstop for the changes TYPO3 cannot see,
|
||||
* such as a referenced content element changing on another page.
|
||||
*/
|
||||
class ContentSyncer
|
||||
{
|
||||
public const PAGES_OUTSIDE_WINDOW = 50;
|
||||
public const REFRESH_AFTER_DAYS = 7;
|
||||
|
||||
private const NIGHT_WINDOW_TIMEZONE = 'Europe/Berlin';
|
||||
private const NIGHT_WINDOW_FROM_HOUR = 1;
|
||||
private const NIGHT_WINDOW_TO_HOUR = 5;
|
||||
private const FLUSH_EVERY_PAGES = 20;
|
||||
|
||||
public function __construct(
|
||||
private readonly ContentFeedClient $client,
|
||||
private readonly ContentDocumentRepository $documentRepository,
|
||||
private readonly ContentSyncStateRepository $stateRepository,
|
||||
private readonly DocumentBodyBuilder $bodyBuilder,
|
||||
private readonly HtmlToMarkdownConverter $converter,
|
||||
private readonly TripCatalog $tripCatalog,
|
||||
private readonly EntityManagerInterface $entityManager,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
}
|
||||
|
||||
public function run(bool $refresh = false, bool $ignoreWindow = false): ContentSyncReport
|
||||
{
|
||||
$report = new ContentSyncReport();
|
||||
$state = $this->stateRepository->get();
|
||||
$this->entityManager->persist($state);
|
||||
$now = $this->clock->now();
|
||||
|
||||
if (null !== $state->getPausedUntil() && $now < $state->getPausedUntil()) {
|
||||
$report->stoppedBy = 'Feed switched off by TYPO3 until '.$state->getPausedUntil()->format('c');
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
$state->setPausedUntil(null)->setLastRunAt($now);
|
||||
$this->client->startRun();
|
||||
|
||||
try {
|
||||
$manifest = $this->client->manifest();
|
||||
$documents = $this->applyManifest($manifest, $report);
|
||||
$state->setManifestGeneratedAt($manifest->generatedAt);
|
||||
$this->entityManager->flush();
|
||||
|
||||
$staleBefore = $refresh ? $now->modify(sprintf('-%d days', self::REFRESH_AFTER_DAYS)) : null;
|
||||
$this->syncRecords($documents, $staleBefore, $report);
|
||||
$this->syncPages($documents, $staleBefore, $ignoreWindow || $this->inNightWindow($now), $report);
|
||||
} catch (ContentFeedUnavailableException $e) {
|
||||
$state->setPausedUntil($e->retryAfter);
|
||||
$report->stoppedBy = $e->getMessage();
|
||||
} catch (ContentFeedStoppedException $e) {
|
||||
$report->stoppedBy = $e->getMessage();
|
||||
}
|
||||
|
||||
if (isset($documents)) {
|
||||
$this->merge();
|
||||
}
|
||||
|
||||
$state->setLastError($report->stoppedBy ?? $report->pagePhaseEndedBy);
|
||||
$this->entityManager->flush();
|
||||
|
||||
if (null === $report->stoppedBy) {
|
||||
$this->logger->info('Content feed sync finished', $report->toLogContext());
|
||||
} else {
|
||||
$this->logger->warning('Content feed sync stopped', $report->toLogContext());
|
||||
}
|
||||
|
||||
if ($report->isComplete() && $report->changedAnything()) {
|
||||
$this->tripCatalog->logUnmatchedCodes();
|
||||
}
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes what the manifest no longer lists and brings every listed item's metadata up
|
||||
* to date. Content is not touched here.
|
||||
*
|
||||
* @return array<string, array{document: ContentDocument, item: ManifestItem}>
|
||||
*/
|
||||
private function applyManifest(ContentManifest $manifest, ContentSyncReport $report): array
|
||||
{
|
||||
$stored = $this->documentRepository->findAllByKey();
|
||||
|
||||
foreach (array_diff_key($stored, $manifest->items) as $document) {
|
||||
$this->entityManager->remove($document);
|
||||
++$report->deleted;
|
||||
}
|
||||
|
||||
$documents = [];
|
||||
|
||||
foreach ($manifest->items as $key => $item) {
|
||||
$document = $stored[$key] ?? null;
|
||||
|
||||
if (null === $document) {
|
||||
$document = new ContentDocument($item->type, $item->uid, $item->language, $item->modified);
|
||||
$this->entityManager->persist($document);
|
||||
++$report->added;
|
||||
}
|
||||
|
||||
$document->setModified($item->modified)->setSite($item->site)->setUrl($item->url);
|
||||
|
||||
if ($document->isPage()) {
|
||||
$document
|
||||
->setTitle($item->title ?? '')
|
||||
->setBreadcrumb([] === $item->breadcrumb ? null : implode(' › ', $item->breadcrumb))
|
||||
->setData(['contentUrl' => $item->contentUrl, 'description' => $item->description, 'detailOf' => $item->detailOf]);
|
||||
}
|
||||
|
||||
$documents[$key] = ['document' => $document, 'item' => $item];
|
||||
}
|
||||
|
||||
return $documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*/
|
||||
private function syncRecords(array $documents, ?\DateTimeImmutable $staleBefore, ContentSyncReport $report): void
|
||||
{
|
||||
/** @var array<string, array<int, list<ContentDocument>>> $stale type => uid => documents; the records endpoint takes no language */
|
||||
$stale = [];
|
||||
|
||||
foreach ($this->dueForFetch($documents, $staleBefore, pages: false) as $document) {
|
||||
$stale[$document->getType()][$document->getUid()][] = $document;
|
||||
}
|
||||
|
||||
foreach ($stale as $type => $byUid) {
|
||||
foreach (array_chunk(array_keys($byUid), ContentFeedClient::RECORDS_BATCH_SIZE) as $uids) {
|
||||
try {
|
||||
$records = $this->client->records($type, $uids);
|
||||
} catch (ContentRequestFailedException $e) {
|
||||
$this->logger->warning('Content feed: records batch failed, retried next run', ['type' => $type, 'error' => $e->getMessage()]);
|
||||
$report->failed += count($uids);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($uids as $uid) {
|
||||
foreach ($byUid[$uid] as $document) {
|
||||
$record = $records[$uid] ?? null;
|
||||
|
||||
if (null === $record) {
|
||||
// Requested but absent: no longer eligible.
|
||||
$this->entityManager->remove($document);
|
||||
++$report->deleted;
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->applyRecord($document, $record);
|
||||
++$report->records;
|
||||
}
|
||||
}
|
||||
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
private function applyRecord(ContentDocument $document, array $record): void
|
||||
{
|
||||
$title = $this->bodyBuilder->title($document->getType(), $record);
|
||||
|
||||
$document
|
||||
->setData($record)
|
||||
->setTitle('' !== $title ? $title : $document->getTitle())
|
||||
// The merged pages are appended in merge().
|
||||
->setBodyMd($this->bodyBuilder->build($document->getType(), $record))
|
||||
->markFetched(ContentFetchState::Ok, $this->clock->now());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
private function syncPages(array $documents, ?\DateTimeImmutable $staleBefore, bool $unlimited, ContentSyncReport $report): void
|
||||
{
|
||||
$due = $this->dueForFetch($documents, $staleBefore, pages: true);
|
||||
|
||||
if (!$unlimited && count($due) > self::PAGES_OUTSIDE_WINDOW) {
|
||||
$report->deferred = count($due) - self::PAGES_OUTSIDE_WINDOW;
|
||||
$due = array_slice($due, 0, self::PAGES_OUTSIDE_WINDOW);
|
||||
}
|
||||
|
||||
$fetched = 0;
|
||||
|
||||
try {
|
||||
foreach ($due as $document) {
|
||||
$this->fetchPage($document, $report);
|
||||
|
||||
if (0 === ++$fetched % self::FLUSH_EVERY_PAGES) {
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
} catch (PagePhaseEndedException $e) {
|
||||
$report->pagePhaseEndedBy = $e->getMessage();
|
||||
$this->logger->warning('Content feed: page phase ended early', ['reason' => $e->getMessage()]);
|
||||
} finally {
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
|
||||
private function fetchPage(ContentDocument $document, ContentSyncReport $report): void
|
||||
{
|
||||
$contentUrl = $document->getData()['contentUrl'] ?? null;
|
||||
|
||||
if (!is_string($contentUrl)) {
|
||||
$document->markFetched(ContentFetchState::Failed, $this->clock->now());
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$result = $this->client->page($contentUrl);
|
||||
|
||||
if ($result->transient) {
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!$result->isOk()) {
|
||||
$this->logger->notice('Content feed: page failed, not refetched until it changes', [
|
||||
'uid' => $document->getUid(),
|
||||
'contentUrl' => $contentUrl,
|
||||
'status' => $result->failedStatus,
|
||||
]);
|
||||
$document->markFetched(ContentFetchState::Failed, $this->clock->now());
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$markdown = $this->converter->convert($result->html, $document->getUrl() ?? $contentUrl);
|
||||
$document->setBodyMd($markdown)->markFetched('' === $markdown ? ContentFetchState::Empty : ContentFetchState::Ok, $this->clock->now());
|
||||
++$report->pages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Items whose `modified` changed since their last fetch, plus, on a refresh, those
|
||||
* fetched before `$staleBefore`, oldest first.
|
||||
*
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*
|
||||
* @return list<ContentDocument>
|
||||
*/
|
||||
private function dueForFetch(array $documents, ?\DateTimeImmutable $staleBefore, bool $pages): array
|
||||
{
|
||||
$due = [];
|
||||
|
||||
foreach ($documents as ['document' => $document]) {
|
||||
if ($document->isPage() !== $pages || !$this->entityManager->contains($document)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$outdated = null !== $staleBefore && (null === $document->getSyncedAt() || $document->getSyncedAt() < $staleBefore);
|
||||
|
||||
if ($document->isStale() || $outdated) {
|
||||
$due[] = $document;
|
||||
}
|
||||
}
|
||||
|
||||
usort($due, static fn (ContentDocument $a, ContentDocument $b): int => ($a->getSyncedAt()?->getTimestamp() ?? 0) <=> ($b->getSyncedAt()?->getTimestamp() ?? 0));
|
||||
|
||||
return $due;
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends every detail page to the record it is the detail page of, and keeps the page
|
||||
* itself out of the index. Recomputed from scratch on every run, so a change or deletion
|
||||
* on either side re-merges.
|
||||
*/
|
||||
private function merge(): void
|
||||
{
|
||||
$documents = $this->documentRepository->findAllByKey();
|
||||
/** @var array<string, list<ContentDocument>> $pagesByRecord */
|
||||
$pagesByRecord = [];
|
||||
|
||||
foreach ($documents as $page) {
|
||||
if (!$page->isPage()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$target = null;
|
||||
foreach ($page->getData()['detailOf'] ?? [] as $ref) {
|
||||
$candidate = $documents[ContentDocument::key($ref['type'], $ref['uid'], $page->getLanguage())] ?? null;
|
||||
|
||||
if (null !== $candidate && ContentFetchState::Ok === $candidate->getFetchState()) {
|
||||
$target = $candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$page->setMergedInto($target?->getRecordRef());
|
||||
|
||||
if (null !== $target && ContentFetchState::Ok === $page->getFetchState() && '' !== $page->getBodyMd()) {
|
||||
$pagesByRecord[$target->getKey()][] = $page;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($documents as $record) {
|
||||
if ($record->isPage() || ContentFetchState::Ok !== $record->getFetchState() || null === $record->getData()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$pages = $pagesByRecord[$record->getKey()] ?? [];
|
||||
usort($pages, static fn (ContentDocument $a, ContentDocument $b): int => $a->getUid() <=> $b->getUid());
|
||||
|
||||
$body = $this->bodyBuilder->build($record->getType(), $record->getData());
|
||||
foreach ($pages as $page) {
|
||||
$body .= "\n\n".$page->getBodyMd();
|
||||
}
|
||||
|
||||
$record->setBodyMd(trim($body));
|
||||
|
||||
// A record without a reachable page of its own is linkable through its detail page.
|
||||
if (null === $record->getUrl() && [] !== $pages) {
|
||||
$record->setUrl($pages[0]->getUrl())->setSite($record->getSite() ?? $pages[0]->getSite());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function inNightWindow(\DateTimeImmutable $now): bool
|
||||
{
|
||||
$hour = (int) $now->setTimezone(new \DateTimeZone(self::NIGHT_WINDOW_TIMEZONE))->format('G');
|
||||
|
||||
return $hour >= self::NIGHT_WINDOW_FROM_HOUR && $hour < self::NIGHT_WINDOW_TO_HOUR;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
/**
|
||||
* Builds the searchable Markdown body and the title of a feed record.
|
||||
*
|
||||
* Every record follows the same pattern: the name as heading, the short introductory
|
||||
* fields as paragraphs, then each `text` field as a section under a German heading, in a
|
||||
* fixed order. Structured facts that people ask about (WLAN, Sauna, Pistenkilometer) are
|
||||
* rendered as a list as well, so the full-text search finds them.
|
||||
*/
|
||||
class DocumentBodyBuilder
|
||||
{
|
||||
/**
|
||||
* Introductory fields, rendered without a heading. Plain text in the feed.
|
||||
*/
|
||||
private const LEAD_FIELDS = ['subline', 'headline'];
|
||||
|
||||
/**
|
||||
* `text` fields per type, in output order, with their section heading. A null heading
|
||||
* renders the field as an introduction straight under the title.
|
||||
*/
|
||||
private const TEXT_SECTIONS = [
|
||||
'product' => [
|
||||
'teaser' => null,
|
||||
'teaserLong' => 'Beschreibung',
|
||||
'feature' => 'Highlights',
|
||||
'conceptDescription' => 'Reisekonzept',
|
||||
'boardDescription' => 'Verpflegung',
|
||||
'includedServices' => 'Leistungen',
|
||||
'additionalServices' => 'Zusatzleistungen',
|
||||
'programmeDescription' => 'Programm',
|
||||
'skiPassDescription' => 'Skipass',
|
||||
'nonBookableInfo' => 'Hinweise zur Buchbarkeit',
|
||||
],
|
||||
'hotel' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'features' => 'Ausstattung',
|
||||
'roomTypes' => 'Zimmer',
|
||||
'additionalInformation' => 'Weitere Informationen',
|
||||
],
|
||||
'region' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'feature' => 'Highlights',
|
||||
'skiArea' => 'Skigebiet',
|
||||
'skiAreaExtended' => 'Skigebiet im Detail',
|
||||
'skiPass' => 'Skipass',
|
||||
'news' => 'Neuigkeiten',
|
||||
],
|
||||
'country' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
],
|
||||
'city' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
'party' => 'Party',
|
||||
'leisure' => 'Freizeit',
|
||||
],
|
||||
'concept' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
'popupText' => 'Kurz erklärt',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Facts per type, with their German label.
|
||||
*/
|
||||
private const FACTS = [
|
||||
'hotel' => [
|
||||
'bedsCount' => 'Betten',
|
||||
'parkingCount' => 'Parkplätze',
|
||||
'altitude' => 'Höhenlage',
|
||||
'distanceBus' => 'Entfernung zur Bushaltestelle',
|
||||
'distanceLift' => 'Entfernung zum Lift',
|
||||
'shower' => 'Dusche/WC',
|
||||
'partyroom' => 'Partyraum',
|
||||
'gamesroom' => 'Spieleraum',
|
||||
'sauna' => 'Sauna',
|
||||
'wifi' => 'WLAN',
|
||||
],
|
||||
'region' => [
|
||||
'length' => 'Pistenkilometer',
|
||||
'height' => 'Höhe',
|
||||
'lifts' => 'Lifte',
|
||||
],
|
||||
];
|
||||
|
||||
public function __construct(private readonly HtmlToMarkdownConverter $converter)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
public function title(string $type, array $record): string
|
||||
{
|
||||
$title = 'news' === $type ? ($record['title'] ?? null) : ($record['name'] ?? null);
|
||||
|
||||
return is_string($title) ? trim($title) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
public function build(string $type, array $record): string
|
||||
{
|
||||
$baseUrl = is_string($record['url'] ?? null) ? $record['url'] : null;
|
||||
$blocks = [];
|
||||
|
||||
$title = $this->title($type, $record);
|
||||
if ('' !== $title) {
|
||||
$blocks[] = '# '.$title;
|
||||
}
|
||||
|
||||
if ('news' === $type) {
|
||||
return $this->join(array_merge($blocks, $this->newsBlocks($record, $baseUrl)));
|
||||
}
|
||||
|
||||
foreach (self::LEAD_FIELDS as $field) {
|
||||
$value = $this->plain($record[$field] ?? null);
|
||||
if ('' !== $value && $value !== $title) {
|
||||
$blocks[] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
if ('hotel' === $type) {
|
||||
$address = $this->plain($record['address'] ?? null);
|
||||
if ('' !== $address) {
|
||||
$blocks[] = 'Adresse: '.$address;
|
||||
}
|
||||
}
|
||||
|
||||
$text = is_array($record['text'] ?? null) ? $record['text'] : [];
|
||||
foreach (self::TEXT_SECTIONS[$type] ?? [] as $field => $heading) {
|
||||
$markdown = $this->converter->convert(is_string($text[$field] ?? null) ? $text[$field] : null, $baseUrl);
|
||||
if ('' === $markdown) {
|
||||
continue;
|
||||
}
|
||||
$blocks[] = null === $heading ? $markdown : "## {$heading}\n\n{$markdown}";
|
||||
}
|
||||
|
||||
$facts = $this->facts($type, is_array($record['facts'] ?? null) ? $record['facts'] : []);
|
||||
if ('' !== $facts) {
|
||||
$blocks[] = "## Fakten\n\n".$facts;
|
||||
}
|
||||
|
||||
if ('region' === $type && is_string($record['webcam'] ?? null) && '' !== trim($record['webcam'])) {
|
||||
$blocks[] = 'Webcam: '.trim($record['webcam']);
|
||||
}
|
||||
|
||||
if ('product' === $type) {
|
||||
$faqs = $this->faqs(is_array($record['faqs'] ?? null) ? $record['faqs'] : [], $baseUrl);
|
||||
if ('' !== $faqs) {
|
||||
$blocks[] = "## Häufige Fragen\n\n".$faqs;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->join($blocks);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function newsBlocks(array $record, ?string $baseUrl): array
|
||||
{
|
||||
$blocks = [];
|
||||
|
||||
if (is_string($record['datetime'] ?? null) && false !== ($date = date_create_immutable($record['datetime']))) {
|
||||
$blocks[] = 'Veröffentlicht am '.$date->format('d.m.Y');
|
||||
}
|
||||
|
||||
foreach (['teaser', 'bodytext'] as $field) {
|
||||
$markdown = $this->converter->convert(is_string($record[$field] ?? null) ? $record[$field] : null, $baseUrl);
|
||||
if ('' !== $markdown) {
|
||||
$blocks[] = $markdown;
|
||||
}
|
||||
}
|
||||
|
||||
$tags = $this->labels($record['tags'] ?? null);
|
||||
if ([] !== $tags) {
|
||||
$blocks[] = 'Themen: '.implode(', ', $tags);
|
||||
}
|
||||
|
||||
return $blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $facts
|
||||
*/
|
||||
private function facts(string $type, array $facts): string
|
||||
{
|
||||
$lines = [];
|
||||
|
||||
foreach (self::FACTS[$type] ?? [] as $field => $label) {
|
||||
$value = $facts[$field] ?? null;
|
||||
|
||||
$rendered = match (true) {
|
||||
is_bool($value) => $value ? 'ja' : 'nein',
|
||||
is_int($value), is_float($value) => (string) $value,
|
||||
is_string($value) => $this->plain($value),
|
||||
default => '',
|
||||
};
|
||||
|
||||
if ('' !== $rendered) {
|
||||
$lines[] = "- {$label}: {$rendered}";
|
||||
}
|
||||
}
|
||||
|
||||
return implode("\n", $lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<mixed> $faqs
|
||||
*/
|
||||
private function faqs(array $faqs, ?string $baseUrl): string
|
||||
{
|
||||
$blocks = [];
|
||||
|
||||
foreach ($faqs as $faq) {
|
||||
if (!is_array($faq)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$question = $this->plain($faq['question'] ?? null);
|
||||
$answer = $this->converter->convert(is_string($faq['answer'] ?? null) ? $faq['answer'] : null, $baseUrl);
|
||||
|
||||
if ('' !== $question && '' !== $answer) {
|
||||
$blocks[] = "### {$question}\n\n{$answer}";
|
||||
}
|
||||
}
|
||||
|
||||
return $this->join($blocks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tags and categories may arrive as strings or as objects with a title or name.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function labels(mixed $values): array
|
||||
{
|
||||
if (!is_array($values)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$labels = [];
|
||||
foreach ($values as $value) {
|
||||
$label = is_array($value) ? ($value['title'] ?? $value['name'] ?? null) : $value;
|
||||
if (is_string($label) && '' !== trim($label)) {
|
||||
$labels[] = trim($label);
|
||||
}
|
||||
}
|
||||
|
||||
return $labels;
|
||||
}
|
||||
|
||||
/**
|
||||
* A field that is plain text in the feed, defensively stripped of any markup.
|
||||
*/
|
||||
private function plain(mixed $value): string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return trim((string) preg_replace('/\s+/u', ' ', html_entity_decode(strip_tags($value), \ENT_QUOTES | \ENT_HTML5, 'UTF-8')));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<string> $blocks
|
||||
*/
|
||||
private function join(array $blocks): string
|
||||
{
|
||||
return implode("\n\n", $blocks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* Ends the whole sync run, keeping everything as it is. The next scheduled run tries again.
|
||||
*
|
||||
* Thrown for a manifest that cannot be trusted, three consecutive 5xx responses or
|
||||
* timeouts, and anything else that means TYPO3 is not in a state to be read.
|
||||
*/
|
||||
class ContentFeedStoppedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* TYPO3 answered 503: the feed has been switched off on its side.
|
||||
*/
|
||||
class ContentFeedUnavailableException extends ContentFeedStoppedException
|
||||
{
|
||||
public function __construct(public readonly ?\DateTimeImmutable $retryAfter)
|
||||
{
|
||||
parent::__construct('TYPO3 switched the content feed off (503)'.(null !== $retryAfter ? ', retry after '.$retryAfter->format('c') : ''));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* One request failed without ending the run. The items it was for keep their stored
|
||||
* version and are retried next run.
|
||||
*/
|
||||
class ContentRequestFailedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* Three slow responses in a row: TYPO3 is under load, so no more pages this run. The rest
|
||||
* of the run (bookkeeping, merging) still completes.
|
||||
*/
|
||||
class PagePhaseEndedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use League\HTMLToMarkdown\Converter\TableConverter;
|
||||
use League\HTMLToMarkdown\HtmlConverter;
|
||||
use Symfony\Component\DomCrawler\UriResolver;
|
||||
|
||||
/**
|
||||
* Turns the CMS's HTML, record text fields and page fragments alike, into Markdown for the
|
||||
* index and the tool output. LLMs read Markdown well; HTML noise only costs tokens.
|
||||
*
|
||||
* Embeds, media and forms are dropped with their content. Relative links are made
|
||||
* absolute against the document's own URL, since the page fragment may carry them.
|
||||
*/
|
||||
class HtmlToMarkdownConverter
|
||||
{
|
||||
private const REMOVED_ELEMENTS = ['script', 'style', 'noscript', 'iframe', 'img', 'picture', 'svg', 'form'];
|
||||
|
||||
private readonly HtmlConverter $converter;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->converter = new HtmlConverter([
|
||||
'header_style' => 'atx',
|
||||
'strip_tags' => true,
|
||||
'strip_placeholder_links' => true,
|
||||
'hard_break' => true,
|
||||
'suppress_errors' => true,
|
||||
'remove_nodes' => implode(' ', self::REMOVED_ELEMENTS),
|
||||
]);
|
||||
$this->converter->getEnvironment()->addConverter(new TableConverter());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $baseUrl what relative links resolve against; left as they are without one
|
||||
*/
|
||||
public function convert(?string $html, ?string $baseUrl = null): string
|
||||
{
|
||||
if (null === $html || '' === trim($html)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$markdown = $this->converter->convert($this->prepare($html, $baseUrl));
|
||||
|
||||
return $this->collapseWhitespace($markdown);
|
||||
}
|
||||
|
||||
private function prepare(string $html, ?string $baseUrl): string
|
||||
{
|
||||
$document = \Dom\HTMLDocument::createFromString('<!DOCTYPE html><html><body>'.$html.'</body></html>', \LIBXML_NOERROR, 'UTF-8');
|
||||
$body = $document->body;
|
||||
|
||||
if (null === $body) {
|
||||
return $html;
|
||||
}
|
||||
|
||||
foreach ($body->querySelectorAll(implode(',', self::REMOVED_ELEMENTS)) as $element) {
|
||||
$element->remove();
|
||||
}
|
||||
|
||||
// Whitespace in text collapses as a browser would; the template's indentation would
|
||||
// otherwise end up inside link labels and headings.
|
||||
foreach ((new \Dom\XPath($document))->query('//body//text()[not(ancestor::pre)]') as $text) {
|
||||
if (!$text instanceof \Dom\Text) {
|
||||
continue;
|
||||
}
|
||||
$text->data = (string) preg_replace('/[ \t\r\n]+/', ' ', $text->data);
|
||||
}
|
||||
|
||||
if (null !== $baseUrl) {
|
||||
foreach ($body->querySelectorAll('a[href]') as $link) {
|
||||
$href = trim((string) $link->getAttribute('href'));
|
||||
|
||||
if ('' === $href || preg_match('/^(mailto|tel|javascript):/i', $href)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$link->setAttribute('href', UriResolver::resolve($href, $baseUrl));
|
||||
}
|
||||
}
|
||||
|
||||
return $body->innerHTML;
|
||||
}
|
||||
|
||||
private function collapseWhitespace(string $markdown): string
|
||||
{
|
||||
// The converter leaves text entity-encoded (`&`), which only costs tokens.
|
||||
$markdown = html_entity_decode($markdown, \ENT_QUOTES | \ENT_HTML5, 'UTF-8');
|
||||
$markdown = str_replace(["\r\n", "\r", "\u{00A0}"], ["\n", "\n", ' '], $markdown);
|
||||
|
||||
// Labels padded by the collapsed whitespace: "[ Kontakt ](…)" -> "[Kontakt](…)".
|
||||
$markdown = (string) preg_replace(['/\[ +/', '/ +\]\(/'], ['[', ']('], $markdown);
|
||||
|
||||
$lines = array_map(
|
||||
// Inner runs only: leading spaces carry list nesting.
|
||||
static fn (string $line): string => rtrim((string) preg_replace('/(?<=\S)[ \t]{2,}/', ' ', $line)),
|
||||
explode("\n", $markdown),
|
||||
);
|
||||
|
||||
return trim((string) preg_replace("/\n{3,}/", "\n\n", implode("\n", $lines)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Entity;
|
||||
|
||||
use App\Enum\ContentFetchState;
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
use Doctrine\DBAL\Types\Types;
|
||||
use Doctrine\ORM\Mapping as ORM;
|
||||
|
||||
/**
|
||||
* One item of the TYPO3 content feed, mirrored for the chatbot's MCP tools.
|
||||
*
|
||||
* Identity is TYPO3's `(type, uid, language)`. A record (product, hotel, destination,
|
||||
* concept, news) carries its structured payload in `data` as received; a page carries its
|
||||
* manifest entry there, including `contentUrl` and `detailOf`.
|
||||
*
|
||||
* A page that is the detail page of a record is merged into it: its text is appended to the
|
||||
* record's `bodyMd`, and the page itself names the record in `mergedInto` and stays out of
|
||||
* the index, so a product does not turn up twice with half its text each time.
|
||||
*
|
||||
* `indexed` is derived, never set: only documents that were fetched, have text and are not
|
||||
* merged into another are searched. A product TYPO3 keeps out of its own website search
|
||||
* (`searchable: false`: closed group trips, staff trainings) stays out of this one too.
|
||||
*/
|
||||
#[ORM\Entity(repositoryClass: ContentDocumentRepository::class)]
|
||||
#[ORM\Table(name: 'content_document')]
|
||||
#[ORM\UniqueConstraint(name: 'content_document_identity', columns: ['type', 'uid', 'language'])]
|
||||
#[ORM\Index(name: 'content_document_indexed_site', columns: ['indexed', 'site'])]
|
||||
#[ORM\Index(name: 'content_document_url', columns: ['url'])]
|
||||
#[ORM\Index(name: 'content_document_fulltext', columns: ['title', 'breadcrumb', 'body_md'], flags: ['fulltext'])]
|
||||
class ContentDocument
|
||||
{
|
||||
public const TYPE_PAGE = 'page';
|
||||
public const RECORD_TYPES = ['product', 'hotel', 'region', 'country', 'city', 'concept', 'news'];
|
||||
|
||||
#[ORM\Id]
|
||||
#[ORM\GeneratedValue]
|
||||
#[ORM\Column]
|
||||
private ?int $id = null;
|
||||
|
||||
#[ORM\Column(length: 16)]
|
||||
private string $type;
|
||||
|
||||
#[ORM\Column]
|
||||
private int $uid;
|
||||
|
||||
#[ORM\Column(length: 8)]
|
||||
private string $language;
|
||||
|
||||
/**
|
||||
* Null for a record without a reachable detail page.
|
||||
*/
|
||||
#[ORM\Column(length: 32, nullable: true)]
|
||||
private ?string $site = null;
|
||||
|
||||
/**
|
||||
* The public URL to cite. Null for a record without a reachable detail page.
|
||||
*/
|
||||
#[ORM\Column(length: 768, nullable: true)]
|
||||
private ?string $url = null;
|
||||
|
||||
#[ORM\Column(length: 255)]
|
||||
private string $title = '';
|
||||
|
||||
/**
|
||||
* Pages only, joined with ›.
|
||||
*/
|
||||
#[ORM\Column(length: 512, nullable: true)]
|
||||
private ?string $breadcrumb = null;
|
||||
|
||||
#[ORM\Column(type: Types::TEXT, length: 16777215)]
|
||||
private string $bodyMd = '';
|
||||
|
||||
/**
|
||||
* @var array<string, mixed>|null
|
||||
*/
|
||||
#[ORM\Column(type: Types::JSON, nullable: true)]
|
||||
private ?array $data = null;
|
||||
|
||||
/**
|
||||
* The manifest's `modified`, as last seen.
|
||||
*/
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE)]
|
||||
private \DateTimeImmutable $modified;
|
||||
|
||||
/**
|
||||
* The manifest's `modified` that the stored fetch outcome belongs to. The item is stale
|
||||
* while this differs from `modified`.
|
||||
*/
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
|
||||
private ?\DateTimeImmutable $fetchedModified = null;
|
||||
|
||||
#[ORM\Column(length: 16, enumType: ContentFetchState::class)]
|
||||
private ContentFetchState $fetchState = ContentFetchState::Pending;
|
||||
|
||||
/**
|
||||
* Last fetch attempt that produced an outcome, whichever it was.
|
||||
*/
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
|
||||
private ?\DateTimeImmutable $syncedAt = null;
|
||||
|
||||
/**
|
||||
* `type:uid` of the record this page is the detail page of, in the same language.
|
||||
*/
|
||||
#[ORM\Column(length: 32, nullable: true)]
|
||||
private ?string $mergedInto = null;
|
||||
|
||||
#[ORM\Column]
|
||||
private bool $indexed = false;
|
||||
|
||||
public function __construct(string $type, int $uid, string $language, \DateTimeImmutable $modified)
|
||||
{
|
||||
$this->type = $type;
|
||||
$this->uid = $uid;
|
||||
$this->language = $language;
|
||||
$this->modified = $modified;
|
||||
}
|
||||
|
||||
public static function key(string $type, int $uid, string $language): string
|
||||
{
|
||||
return $type.':'.$uid.':'.$language;
|
||||
}
|
||||
|
||||
public function getKey(): string
|
||||
{
|
||||
return self::key($this->type, $this->uid, $this->language);
|
||||
}
|
||||
|
||||
/**
|
||||
* The `type:uid` a merged page refers to its record by.
|
||||
*/
|
||||
public function getRecordRef(): string
|
||||
{
|
||||
return $this->type.':'.$this->uid;
|
||||
}
|
||||
|
||||
public function isPage(): bool
|
||||
{
|
||||
return self::TYPE_PAGE === $this->type;
|
||||
}
|
||||
|
||||
public function isStale(): bool
|
||||
{
|
||||
return null === $this->fetchedModified || $this->fetchedModified != $this->modified;
|
||||
}
|
||||
|
||||
public function getId(): ?int
|
||||
{
|
||||
return $this->id;
|
||||
}
|
||||
|
||||
public function getType(): string
|
||||
{
|
||||
return $this->type;
|
||||
}
|
||||
|
||||
public function getUid(): int
|
||||
{
|
||||
return $this->uid;
|
||||
}
|
||||
|
||||
public function getLanguage(): string
|
||||
{
|
||||
return $this->language;
|
||||
}
|
||||
|
||||
public function getSite(): ?string
|
||||
{
|
||||
return $this->site;
|
||||
}
|
||||
|
||||
public function setSite(?string $site): self
|
||||
{
|
||||
$this->site = $site;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getUrl(): ?string
|
||||
{
|
||||
return $this->url;
|
||||
}
|
||||
|
||||
public function setUrl(?string $url): self
|
||||
{
|
||||
$this->url = $url;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getTitle(): string
|
||||
{
|
||||
return $this->title;
|
||||
}
|
||||
|
||||
public function setTitle(string $title): self
|
||||
{
|
||||
$this->title = mb_substr($title, 0, 255);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getBreadcrumb(): ?string
|
||||
{
|
||||
return $this->breadcrumb;
|
||||
}
|
||||
|
||||
public function setBreadcrumb(?string $breadcrumb): self
|
||||
{
|
||||
$this->breadcrumb = null === $breadcrumb ? null : mb_substr($breadcrumb, 0, 512);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getBodyMd(): string
|
||||
{
|
||||
return $this->bodyMd;
|
||||
}
|
||||
|
||||
public function setBodyMd(string $bodyMd): self
|
||||
{
|
||||
$this->bodyMd = $bodyMd;
|
||||
$this->updateIndexed();
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>|null
|
||||
*/
|
||||
public function getData(): ?array
|
||||
{
|
||||
return $this->data;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed>|null $data
|
||||
*/
|
||||
public function setData(?array $data): self
|
||||
{
|
||||
$this->data = $data;
|
||||
$this->updateIndexed();
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getModified(): \DateTimeImmutable
|
||||
{
|
||||
return $this->modified;
|
||||
}
|
||||
|
||||
public function setModified(\DateTimeImmutable $modified): self
|
||||
{
|
||||
// Compared by value: assigning an equal instance would still schedule an update.
|
||||
if ($modified != $this->modified) {
|
||||
$this->modified = $modified;
|
||||
}
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getFetchedModified(): ?\DateTimeImmutable
|
||||
{
|
||||
return $this->fetchedModified;
|
||||
}
|
||||
|
||||
public function getFetchState(): ContentFetchState
|
||||
{
|
||||
return $this->fetchState;
|
||||
}
|
||||
|
||||
public function getSyncedAt(): ?\DateTimeImmutable
|
||||
{
|
||||
return $this->syncedAt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the outcome of a fetch against the `modified` it was made for.
|
||||
*/
|
||||
public function markFetched(ContentFetchState $state, \DateTimeImmutable $at): self
|
||||
{
|
||||
$this->fetchState = $state;
|
||||
$this->fetchedModified = $this->modified;
|
||||
$this->syncedAt = $at;
|
||||
$this->updateIndexed();
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getMergedInto(): ?string
|
||||
{
|
||||
return $this->mergedInto;
|
||||
}
|
||||
|
||||
public function setMergedInto(?string $mergedInto): self
|
||||
{
|
||||
$this->mergedInto = $mergedInto;
|
||||
$this->updateIndexed();
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function isIndexed(): bool
|
||||
{
|
||||
return $this->indexed;
|
||||
}
|
||||
|
||||
private function updateIndexed(): void
|
||||
{
|
||||
$this->indexed = ContentFetchState::Ok === $this->fetchState
|
||||
&& '' !== $this->bodyMd
|
||||
&& null === $this->mergedInto
|
||||
&& !$this->isHiddenFromSearch();
|
||||
}
|
||||
|
||||
/**
|
||||
* TYPO3's own "not in the website search" flag, on products.
|
||||
*/
|
||||
public function isHiddenFromSearch(): bool
|
||||
{
|
||||
return 'product' === $this->type && false === ($this->data['searchable'] ?? null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Entity;
|
||||
|
||||
use App\Repository\ContentSyncStateRepository;
|
||||
use Doctrine\DBAL\Types\Types;
|
||||
use Doctrine\ORM\Mapping as ORM;
|
||||
|
||||
/**
|
||||
* Bookkeeping for the content feed sync. A single row.
|
||||
*
|
||||
* `manifestGeneratedAt` is the freshness figure: the `generatedAt` of the last manifest
|
||||
* that was accepted. A run that stops early still advances it, because the deletions and
|
||||
* the stale marks it is based on were applied.
|
||||
*/
|
||||
#[ORM\Entity(repositoryClass: ContentSyncStateRepository::class)]
|
||||
#[ORM\Table(name: 'content_sync_state')]
|
||||
class ContentSyncState
|
||||
{
|
||||
#[ORM\Id]
|
||||
#[ORM\GeneratedValue]
|
||||
#[ORM\Column]
|
||||
private ?int $id = null;
|
||||
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
|
||||
private ?\DateTimeImmutable $manifestGeneratedAt = null;
|
||||
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
|
||||
private ?\DateTimeImmutable $lastRunAt = null;
|
||||
|
||||
/**
|
||||
* How the last run ended, in a few words. Null after a run that completed.
|
||||
*/
|
||||
#[ORM\Column(type: Types::TEXT, nullable: true)]
|
||||
private ?string $lastError = null;
|
||||
|
||||
/**
|
||||
* Set from `Retry-After` when TYPO3 switched the feed off. Runs before it do nothing.
|
||||
*/
|
||||
#[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)]
|
||||
private ?\DateTimeImmutable $pausedUntil = null;
|
||||
|
||||
public function getId(): ?int
|
||||
{
|
||||
return $this->id;
|
||||
}
|
||||
|
||||
public function getManifestGeneratedAt(): ?\DateTimeImmutable
|
||||
{
|
||||
return $this->manifestGeneratedAt;
|
||||
}
|
||||
|
||||
public function setManifestGeneratedAt(?\DateTimeImmutable $manifestGeneratedAt): self
|
||||
{
|
||||
$this->manifestGeneratedAt = $manifestGeneratedAt;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getLastRunAt(): ?\DateTimeImmutable
|
||||
{
|
||||
return $this->lastRunAt;
|
||||
}
|
||||
|
||||
public function setLastRunAt(?\DateTimeImmutable $lastRunAt): self
|
||||
{
|
||||
$this->lastRunAt = $lastRunAt;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getLastError(): ?string
|
||||
{
|
||||
return $this->lastError;
|
||||
}
|
||||
|
||||
public function setLastError(?string $lastError): self
|
||||
{
|
||||
$this->lastError = $lastError;
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getPausedUntil(): ?\DateTimeImmutable
|
||||
{
|
||||
return $this->pausedUntil;
|
||||
}
|
||||
|
||||
public function setPausedUntil(?\DateTimeImmutable $pausedUntil): self
|
||||
{
|
||||
$this->pausedUntil = $pausedUntil;
|
||||
|
||||
return $this;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Enum;
|
||||
|
||||
/**
|
||||
* Outcome of the last fetch of one content feed item.
|
||||
*
|
||||
* `Empty` and `Failed` are remembered together with the manifest `modified` they were
|
||||
* reached at, so a page that renders without text or keeps answering 404 is not refetched
|
||||
* every run, only once TYPO3 reports a change or the weekly refresh comes round.
|
||||
*/
|
||||
enum ContentFetchState: string
|
||||
{
|
||||
/** Listed in the manifest, not fetched yet. */
|
||||
case Pending = 'pending';
|
||||
case Ok = 'ok';
|
||||
/** Fetched, but converted to no text. Kept out of the index. */
|
||||
case Empty = 'empty';
|
||||
/** The page answered 404 or 5xx. Kept out of the index. */
|
||||
case Failed = 'failed';
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Model\Chatbot;
|
||||
|
||||
/**
|
||||
* The filters of the search_trips tool, validated.
|
||||
*/
|
||||
final readonly class TripCriteria
|
||||
{
|
||||
public function __construct(
|
||||
public ?string $query = null,
|
||||
public ?\DateTimeImmutable $dateFrom = null,
|
||||
public ?\DateTimeImmutable $dateTo = null,
|
||||
public ?string $country = null,
|
||||
public ?string $region = null,
|
||||
public ?string $concept = null,
|
||||
public ?float $maxPrice = null,
|
||||
public ?int $persons = null,
|
||||
public ?bool $daytrip = null,
|
||||
public ?string $site = null,
|
||||
public int $limit = 5,
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Model\ContentFeed;
|
||||
|
||||
/**
|
||||
* The complete list of what TYPO3 publishes. An item missing from it has been deleted,
|
||||
* hidden or unpublished.
|
||||
*/
|
||||
final readonly class ContentManifest
|
||||
{
|
||||
/**
|
||||
* @param array<string, ManifestItem> $items keyed by ManifestItem::key()
|
||||
* @param list<string> $siteHosts hosts of the sites' base URLs, the only ones a contentUrl may point to
|
||||
*/
|
||||
public function __construct(
|
||||
public \DateTimeImmutable $generatedAt,
|
||||
public array $items,
|
||||
public array $siteHosts,
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Model\ContentFeed;
|
||||
|
||||
/**
|
||||
* What one content sync run did, for the command output and the log.
|
||||
*/
|
||||
final class ContentSyncReport
|
||||
{
|
||||
public int $added = 0;
|
||||
public int $deleted = 0;
|
||||
public int $records = 0;
|
||||
public int $pages = 0;
|
||||
/** Records or pages that failed this run. */
|
||||
public int $failed = 0;
|
||||
/** Pages left for a later run by the limit outside the night window. */
|
||||
public int $deferred = 0;
|
||||
/** Why the run ended early, with nothing after that point fetched. */
|
||||
public ?string $stoppedBy = null;
|
||||
/** Why the page phase ended early; the rest of the run completed. */
|
||||
public ?string $pagePhaseEndedBy = null;
|
||||
|
||||
public function isComplete(): bool
|
||||
{
|
||||
return null === $this->stoppedBy && null === $this->pagePhaseEndedBy && 0 === $this->deferred;
|
||||
}
|
||||
|
||||
public function changedAnything(): bool
|
||||
{
|
||||
return $this->added + $this->deleted + $this->records + $this->pages > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, int|string|null>
|
||||
*/
|
||||
public function toLogContext(): array
|
||||
{
|
||||
return get_object_vars($this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Model\ContentFeed;
|
||||
|
||||
use App\Entity\ContentDocument;
|
||||
|
||||
/**
|
||||
* One entry of the TYPO3 content manifest. Identity is `(type, uid, language)`.
|
||||
*
|
||||
* `contentUrl`, `breadcrumb` and `detailOf` are set on pages only. `contentUrl` is fetched as
|
||||
* given and never constructed: the trailing-slash variant silently serves the whole page.
|
||||
*/
|
||||
final readonly class ManifestItem
|
||||
{
|
||||
/**
|
||||
* @param list<string> $breadcrumb
|
||||
* @param list<array{type: string, uid: int}> $detailOf
|
||||
*/
|
||||
public function __construct(
|
||||
public string $type,
|
||||
public int $uid,
|
||||
public string $language,
|
||||
public ?string $site,
|
||||
public ?string $url,
|
||||
public \DateTimeImmutable $modified,
|
||||
public ?string $contentUrl = null,
|
||||
public ?string $title = null,
|
||||
public ?string $description = null,
|
||||
public array $breadcrumb = [],
|
||||
public array $detailOf = [],
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $item
|
||||
*/
|
||||
public static function fromArray(array $item): self
|
||||
{
|
||||
if (!is_string($item['type'] ?? null) || !is_int($item['uid'] ?? null) || !is_string($item['language'] ?? null) || !is_string($item['modified'] ?? null)) {
|
||||
throw new \InvalidArgumentException('Manifest item without type, uid, language or modified');
|
||||
}
|
||||
|
||||
$modified = self::localTime($item['modified']);
|
||||
if (null === $modified) {
|
||||
throw new \InvalidArgumentException(sprintf('Manifest item %s:%d has an unparseable modified', $item['type'], $item['uid']));
|
||||
}
|
||||
|
||||
$detailOf = [];
|
||||
foreach (is_array($item['detailOf'] ?? null) ? $item['detailOf'] : [] as $ref) {
|
||||
if (is_array($ref) && is_string($ref['type'] ?? null) && is_int($ref['uid'] ?? null)) {
|
||||
$detailOf[] = ['type' => $ref['type'], 'uid' => $ref['uid']];
|
||||
}
|
||||
}
|
||||
|
||||
$string = static fn (mixed $value): ?string => is_string($value) && '' !== $value ? $value : null;
|
||||
|
||||
return new self(
|
||||
type: $item['type'],
|
||||
uid: $item['uid'],
|
||||
language: $item['language'],
|
||||
site: $string($item['site'] ?? null),
|
||||
url: $string($item['url'] ?? null),
|
||||
modified: $modified,
|
||||
contentUrl: $string($item['contentUrl'] ?? null),
|
||||
title: $string($item['title'] ?? null),
|
||||
description: $string($item['description'] ?? null),
|
||||
breadcrumb: array_values(array_filter(is_array($item['breadcrumb'] ?? null) ? $item['breadcrumb'] : [], 'is_string')),
|
||||
detailOf: $detailOf,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A feed timestamp in the application's timezone. Doctrine stores datetimes without
|
||||
* their offset, so a value kept in TYPO3's +02:00 would never compare equal to itself
|
||||
* once reloaded, and every item would look changed on every run.
|
||||
*/
|
||||
public static function localTime(string $value): ?\DateTimeImmutable
|
||||
{
|
||||
$date = date_create_immutable($value);
|
||||
|
||||
return false === $date ? null : $date->setTimezone(new \DateTimeZone(date_default_timezone_get()));
|
||||
}
|
||||
|
||||
public function key(): string
|
||||
{
|
||||
return ContentDocument::key($this->type, $this->uid, $this->language);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Model\ContentFeed;
|
||||
|
||||
/**
|
||||
* Outcome of fetching one page's contentUrl.
|
||||
*
|
||||
* A 404 or 5xx is a failure of that page, remembered against its `modified`. A timeout is
|
||||
* transient: the page keeps its stored version and is retried next run.
|
||||
*/
|
||||
final readonly class PageFetchResult
|
||||
{
|
||||
private function __construct(
|
||||
public ?string $html,
|
||||
public ?int $failedStatus,
|
||||
public bool $transient,
|
||||
) {
|
||||
}
|
||||
|
||||
public static function ok(string $html): self
|
||||
{
|
||||
return new self($html, null, false);
|
||||
}
|
||||
|
||||
public static function failed(int $status): self
|
||||
{
|
||||
return new self(null, $status, false);
|
||||
}
|
||||
|
||||
public static function transient(): self
|
||||
{
|
||||
return new self(null, null, true);
|
||||
}
|
||||
|
||||
public function isOk(): bool
|
||||
{
|
||||
return null !== $this->html;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Repository;
|
||||
|
||||
use App\Entity\ContentDocument;
|
||||
use App\Enum\ContentFetchState;
|
||||
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
|
||||
use Doctrine\DBAL\ArrayParameterType;
|
||||
use Doctrine\DBAL\ParameterType;
|
||||
use Doctrine\Persistence\ManagerRegistry;
|
||||
|
||||
/**
|
||||
* @extends ServiceEntityRepository<ContentDocument>
|
||||
*/
|
||||
class ContentDocumentRepository extends ServiceEntityRepository
|
||||
{
|
||||
public function __construct(ManagerRegistry $registry)
|
||||
{
|
||||
parent::__construct($registry, ContentDocument::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, ContentDocument> keyed by ContentDocument::getKey()
|
||||
*/
|
||||
public function findAllByKey(): array
|
||||
{
|
||||
$documents = [];
|
||||
|
||||
foreach ($this->findAll() as $document) {
|
||||
$documents[$document->getKey()] = $document;
|
||||
}
|
||||
|
||||
return $documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-text search over the indexed documents that can be linked to.
|
||||
*
|
||||
* DBAL rather than DQL: Doctrine has no MATCH ... AGAINST. The score is weighted by type
|
||||
* so that, at similar relevance, what people book ranks above what they read.
|
||||
*
|
||||
* @param string $booleanQuery a MySQL boolean-mode query, built by ContentSearchQuery
|
||||
* @param list<string> $types empty for all
|
||||
* @param array<string, float> $typeWeights
|
||||
*
|
||||
* @return list<array{type: string, uid: int, title: string, url: string, site: ?string, breadcrumb: ?string, body_md: string, score: float}>
|
||||
*/
|
||||
public function searchFulltext(string $booleanQuery, array $types, ?string $site, array $typeWeights, int $limit): array
|
||||
{
|
||||
$weight = 'CASE type';
|
||||
$params = ['query' => $booleanQuery, 'limit' => $limit];
|
||||
$paramTypes = ['limit' => ParameterType::INTEGER];
|
||||
|
||||
foreach (array_keys($typeWeights) as $i => $type) {
|
||||
$weight .= sprintf(' WHEN :wt%d THEN :wv%d', $i, $i);
|
||||
$params['wt'.$i] = $type;
|
||||
$params['wv'.$i] = $typeWeights[$type];
|
||||
}
|
||||
$weight .= ' ELSE 1 END';
|
||||
|
||||
$sql = 'SELECT type, uid, title, url, site, breadcrumb, body_md,'
|
||||
.' MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE) * '.$weight.' AS score'
|
||||
.' FROM content_document'
|
||||
.' WHERE indexed = 1 AND url IS NOT NULL'
|
||||
.' AND MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE)';
|
||||
|
||||
if ([] !== $types) {
|
||||
$sql .= ' AND type IN (:types)';
|
||||
$params['types'] = $types;
|
||||
$paramTypes['types'] = ArrayParameterType::STRING;
|
||||
}
|
||||
|
||||
if (null !== $site) {
|
||||
$sql .= ' AND site = :site';
|
||||
$params['site'] = $site;
|
||||
}
|
||||
|
||||
$sql .= ' ORDER BY score DESC, uid ASC LIMIT :limit';
|
||||
|
||||
$rows = $this->getEntityManager()->getConnection()->fetchAllAssociative($sql, $params, $paramTypes);
|
||||
|
||||
return array_map(static fn (array $row): array => [
|
||||
'type' => (string) $row['type'],
|
||||
'uid' => (int) $row['uid'],
|
||||
'title' => (string) $row['title'],
|
||||
'url' => (string) $row['url'],
|
||||
'site' => null === $row['site'] ? null : (string) $row['site'],
|
||||
'breadcrumb' => null === $row['breadcrumb'] ? null : (string) $row['breadcrumb'],
|
||||
'body_md' => (string) $row['body_md'],
|
||||
'score' => (float) $row['score'],
|
||||
], $rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* The `(type, uid)` of every fetched record whose text matches, for resolving free text
|
||||
* in the trip search. Unlike searchFulltext(), records without a URL count: a hotel with
|
||||
* no page of its own still leads to the products it belongs to.
|
||||
*
|
||||
* @param list<string> $types
|
||||
*
|
||||
* @return list<array{type: string, uid: int}>
|
||||
*/
|
||||
public function matchRecords(string $booleanQuery, array $types, string $language): array
|
||||
{
|
||||
$rows = $this->getEntityManager()->getConnection()->fetchAllAssociative(
|
||||
'SELECT type, uid FROM content_document'
|
||||
.' WHERE fetch_state = :ok AND language = :language AND type IN (:types)'
|
||||
.' AND MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE)',
|
||||
['ok' => ContentFetchState::Ok->value, 'language' => $language, 'types' => $types, 'query' => $booleanQuery],
|
||||
['types' => ArrayParameterType::STRING],
|
||||
);
|
||||
|
||||
return array_map(static fn (array $row): array => ['type' => (string) $row['type'], 'uid' => (int) $row['uid']], $rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetched records of the given types in one language, keyed by `type:uid`.
|
||||
*
|
||||
* @param list<string> $types
|
||||
*
|
||||
* @return array<string, ContentDocument>
|
||||
*/
|
||||
public function findRecords(array $types, string $language): array
|
||||
{
|
||||
/** @var list<ContentDocument> $documents */
|
||||
$documents = $this->createQueryBuilder('d')
|
||||
->where('d.type IN (:types)')
|
||||
->andWhere('d.language = :language')
|
||||
->andWhere('d.fetchState = :ok')
|
||||
->setParameter('types', $types)
|
||||
->setParameter('language', $language)
|
||||
->setParameter('ok', ContentFetchState::Ok)
|
||||
->getQuery()
|
||||
->getResult();
|
||||
|
||||
$indexed = [];
|
||||
|
||||
foreach ($documents as $document) {
|
||||
$indexed[$document->getRecordRef()] = $document;
|
||||
}
|
||||
|
||||
return $indexed;
|
||||
}
|
||||
|
||||
/**
|
||||
* The indexed document published at a URL. Trailing slashes are not significant.
|
||||
*/
|
||||
public function findOneIndexedByUrl(string $url): ?ContentDocument
|
||||
{
|
||||
$trimmed = rtrim($url, '/');
|
||||
|
||||
/** @var list<ContentDocument> $documents */
|
||||
$documents = $this->createQueryBuilder('d')
|
||||
->where('d.url IN (:urls)')
|
||||
->andWhere('d.indexed = true')
|
||||
->setParameter('urls', [$trimmed, $trimmed.'/'])
|
||||
->orderBy('d.id', 'ASC')
|
||||
->setMaxResults(1)
|
||||
->getQuery()
|
||||
->getResult();
|
||||
|
||||
return $documents[0] ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Repository;
|
||||
|
||||
use App\Entity\ContentSyncState;
|
||||
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
|
||||
use Doctrine\Persistence\ManagerRegistry;
|
||||
|
||||
/**
|
||||
* @extends ServiceEntityRepository<ContentSyncState>
|
||||
*/
|
||||
class ContentSyncStateRepository extends ServiceEntityRepository
|
||||
{
|
||||
public function __construct(ManagerRegistry $registry)
|
||||
{
|
||||
parent::__construct($registry, ContentSyncState::class);
|
||||
}
|
||||
|
||||
/**
|
||||
* The single state row, created unpersisted on first use.
|
||||
*/
|
||||
public function get(): ContentSyncState
|
||||
{
|
||||
return $this->findOneBy([], ['id' => 'ASC']) ?? new ContentSyncState();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Security;
|
||||
|
||||
use Symfony\Component\Security\Core\Exception\BadCredentialsException;
|
||||
use Symfony\Component\Security\Http\AccessToken\AccessTokenHandlerInterface;
|
||||
use Symfony\Component\Security\Http\Authenticator\Passport\Badge\UserBadge;
|
||||
|
||||
/**
|
||||
* Checks the static bearer token n8n sends to the chatbot MCP endpoint.
|
||||
*
|
||||
* The data behind it is public, but the endpoint must not become a free LLM-tooling API
|
||||
* for whoever finds it. A static token rather than the OAuth2 client-credentials grant of
|
||||
* /api: n8n's MCP Client Tool node sends a fixed header and cannot refresh a 10-minute token.
|
||||
* An unconfigured token refuses every request.
|
||||
*/
|
||||
final class McpAccessTokenHandler implements AccessTokenHandlerInterface
|
||||
{
|
||||
public const USER = 'n8n';
|
||||
|
||||
public function __construct(
|
||||
#[\SensitiveParameter]
|
||||
private readonly string $token,
|
||||
) {
|
||||
}
|
||||
|
||||
public function getUserBadgeFrom(#[\SensitiveParameter] string $accessToken): UserBadge
|
||||
{
|
||||
if ('' === $this->token || !hash_equals($this->token, $accessToken)) {
|
||||
throw new BadCredentialsException('Invalid MCP token.');
|
||||
}
|
||||
|
||||
return new UserBadge(self::USER);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user