feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed
This commit is contained in:
fromme committed
1 parent
ff7ddf7a9e
commit
1b4384e3f3
52 files changed
+5117
-4
No files matched your search
@@ -0,0 +1,312 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use App\ContentFeed\Exception\ContentFeedStoppedException;
|
||||
use App\ContentFeed\Exception\ContentFeedUnavailableException;
|
||||
use App\ContentFeed\Exception\ContentRequestFailedException;
|
||||
use App\ContentFeed\Exception\PagePhaseEndedException;
|
||||
use App\Model\ContentFeed\ContentManifest;
|
||||
use App\Model\ContentFeed\ManifestItem;
|
||||
use App\Model\ContentFeed\PageFetchResult;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
|
||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||
use Symfony\Contracts\HttpClient\ResponseInterface;
|
||||
|
||||
/**
|
||||
* Reads the TYPO3 content feed: the manifest, records by uid, and rendered page fragments.
|
||||
*
|
||||
* The website is an old TYPO3 install on which every page fetch is a full frontend render,
|
||||
* so the load rules here are a contract with its maintainers, not tuning:
|
||||
*
|
||||
* - one request at a time, never parallel;
|
||||
* - at most one page fetch per second;
|
||||
* - after a response slower than 2 s, wait 5 s; after three in a row, no more pages this run;
|
||||
* - a 503 (the kill switch) or three consecutive 5xx responses or timeouts end the run.
|
||||
*
|
||||
* One instance serves one run: call startRun() first, it resets the counters.
|
||||
*/
|
||||
class ContentFeedClient
|
||||
{
|
||||
public const RECORDS_BATCH_SIZE = 100;
|
||||
|
||||
private const TIMEOUT_SECONDS = 10.0;
|
||||
private const SLOW_SECONDS = 2.0;
|
||||
private const SLOW_PAUSE_SECONDS = 5.0;
|
||||
private const PAGE_INTERVAL_SECONDS = 1.0;
|
||||
private const MAX_CONSECUTIVE_FAILURES = 3;
|
||||
private const MAX_CONSECUTIVE_SLOW = 3;
|
||||
|
||||
private int $consecutiveFailures = 0;
|
||||
private int $consecutiveSlow = 0;
|
||||
private ?\DateTimeImmutable $notBefore = null;
|
||||
private ?\DateTimeImmutable $lastPageAt = null;
|
||||
|
||||
/** @var list<string> */
|
||||
private array $siteHosts = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly HttpClientInterface $httpClient,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly string $baseUrl,
|
||||
private readonly string $apiKey,
|
||||
) {
|
||||
}
|
||||
|
||||
public function startRun(): void
|
||||
{
|
||||
$this->consecutiveFailures = 0;
|
||||
$this->consecutiveSlow = 0;
|
||||
$this->notBefore = null;
|
||||
$this->lastPageAt = null;
|
||||
$this->siteHosts = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole manifest. Anything but a usable one stops the run: an empty TYPO3 is a bug,
|
||||
* not a deletion.
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function manifest(): ContentManifest
|
||||
{
|
||||
if ('' === $this->apiKey) {
|
||||
throw new ContentFeedStoppedException('APP_CMS_API_KEY is not configured');
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/manifest'), []);
|
||||
|
||||
if (null === $response || 200 !== $response['status']) {
|
||||
throw new ContentFeedStoppedException(sprintf('Manifest request failed (%s)', $response['status'] ?? 'timeout'));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentFeedStoppedException('Manifest is not valid JSON', 0, $e);
|
||||
}
|
||||
|
||||
if (!is_array($data) || !is_string($data['generatedAt'] ?? null) || !is_array($data['items'] ?? null) || [] === $data['items']) {
|
||||
throw new ContentFeedStoppedException('Manifest has no generatedAt or no items');
|
||||
}
|
||||
|
||||
$generatedAt = ManifestItem::localTime($data['generatedAt']);
|
||||
if (null === $generatedAt) {
|
||||
throw new ContentFeedStoppedException('Manifest generatedAt is unparseable');
|
||||
}
|
||||
|
||||
$items = [];
|
||||
foreach ($data['items'] as $raw) {
|
||||
try {
|
||||
$item = ManifestItem::fromArray(is_array($raw) ? $raw : []);
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
// A malformed entry would otherwise read as a deletion of whatever it was.
|
||||
throw new ContentFeedStoppedException('Manifest contains an unusable item: '.$e->getMessage(), 0, $e);
|
||||
}
|
||||
$items[$item->key()] = $item;
|
||||
}
|
||||
|
||||
$hosts = [];
|
||||
foreach (is_array($data['sites'] ?? null) ? $data['sites'] : [] as $site) {
|
||||
$host = is_array($site) && is_string($site['base'] ?? null) ? parse_url($site['base'], \PHP_URL_HOST) : null;
|
||||
if (is_string($host)) {
|
||||
$hosts[] = strtolower($host);
|
||||
}
|
||||
}
|
||||
$this->siteHosts = array_values(array_unique($hosts));
|
||||
|
||||
return new ContentManifest($generatedAt, $items, $this->siteHosts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records of one type by uid, keyed by uid. A requested uid that is absent is no
|
||||
* longer eligible.
|
||||
*
|
||||
* @param list<int> $uids at most RECORDS_BATCH_SIZE
|
||||
*
|
||||
* @return array<int, array<string, mixed>>
|
||||
*
|
||||
* @throws ContentRequestFailedException this batch failed, the run goes on
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function records(string $type, array $uids): array
|
||||
{
|
||||
if (count($uids) > self::RECORDS_BATCH_SIZE) {
|
||||
throw new \InvalidArgumentException(sprintf('At most %d uids per records request', self::RECORDS_BATCH_SIZE));
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/records'), ['type' => $type, 'uids' => implode(',', $uids)]);
|
||||
|
||||
if (null === $response) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s timed out', $type));
|
||||
}
|
||||
|
||||
if (200 !== $response['status']) {
|
||||
if ($response['status'] >= 500) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s failed (%d)', $type, $response['status']));
|
||||
}
|
||||
|
||||
// 403 and the like: misconfiguration, which the next request would hit too.
|
||||
throw new ContentFeedStoppedException(sprintf('Records request for %s answered %d', $type, $response['status']));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s is not valid JSON', $type), 0, $e);
|
||||
}
|
||||
|
||||
// Anything but the documented envelope fails the batch: read as an empty list, it
|
||||
// would delete every record requested.
|
||||
if (!is_array($data) || ($data['type'] ?? null) !== $type || !is_array($data['items'] ?? null) || !array_is_list($data['items'])) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s has no items list', $type));
|
||||
}
|
||||
|
||||
$records = [];
|
||||
foreach ($data['items'] as $record) {
|
||||
if (is_array($record) && is_int($record['uid'] ?? null)) {
|
||||
$records[$record['uid']] = $record;
|
||||
}
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
/**
|
||||
* The bare HTML fragment of one page, fetched at most once a second.
|
||||
*
|
||||
* @throws PagePhaseEndedException
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function page(string $contentUrl): PageFetchResult
|
||||
{
|
||||
$host = parse_url($contentUrl, \PHP_URL_HOST);
|
||||
|
||||
// The key goes with every request, so it only ever goes to a host of the feed's own sites.
|
||||
if (!is_string($host) || !in_array(strtolower($host), $this->siteHosts, true)) {
|
||||
$this->logger->warning('Content feed: contentUrl outside the manifest sites, skipped', ['contentUrl' => $contentUrl]);
|
||||
|
||||
return PageFetchResult::failed(0);
|
||||
}
|
||||
|
||||
if (null !== $this->lastPageAt) {
|
||||
$this->waitUntil($this->lastPageAt->modify(sprintf('+%d ms', (int) (self::PAGE_INTERVAL_SECONDS * 1000))));
|
||||
}
|
||||
$this->lastPageAt = $this->clock->now();
|
||||
|
||||
$response = $this->send($contentUrl, [], countSlow: true);
|
||||
|
||||
if (null === $response) {
|
||||
return PageFetchResult::transient();
|
||||
}
|
||||
|
||||
if (200 === $response['status']) {
|
||||
return PageFetchResult::ok($response['body']);
|
||||
}
|
||||
|
||||
return PageFetchResult::failed($response['status']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends one GET under the load rules. Null on a timeout or transport error.
|
||||
*
|
||||
* @param array<string, string> $query
|
||||
*
|
||||
* @return array{status: int, body: string}|null
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
* @throws PagePhaseEndedException
|
||||
*/
|
||||
private function send(string $url, array $query, bool $countSlow = false): ?array
|
||||
{
|
||||
if (null !== $this->notBefore) {
|
||||
$this->waitUntil($this->notBefore);
|
||||
$this->notBefore = null;
|
||||
}
|
||||
|
||||
$result = null;
|
||||
$duration = 0.0;
|
||||
|
||||
try {
|
||||
$response = $this->httpClient->request('GET', $url, [
|
||||
'query' => $query,
|
||||
'headers' => ['X-Api-Key' => $this->apiKey],
|
||||
'timeout' => self::TIMEOUT_SECONDS,
|
||||
'max_duration' => self::TIMEOUT_SECONDS,
|
||||
]);
|
||||
$result = ['status' => $response->getStatusCode(), 'body' => $response->getContent(false)];
|
||||
$duration = $this->duration($response);
|
||||
} catch (TransportExceptionInterface $e) {
|
||||
$this->logger->warning('Content feed: request failed', ['url' => $url, 'error' => $e->getMessage()]);
|
||||
$duration = self::TIMEOUT_SECONDS;
|
||||
}
|
||||
|
||||
if (null !== $result && 503 === $result['status']) {
|
||||
throw new ContentFeedUnavailableException($this->retryAfter($response));
|
||||
}
|
||||
|
||||
if (null === $result || $result['status'] >= 500) {
|
||||
if (++$this->consecutiveFailures >= self::MAX_CONSECUTIVE_FAILURES) {
|
||||
throw new ContentFeedStoppedException(sprintf('%d consecutive 5xx responses or timeouts', self::MAX_CONSECUTIVE_FAILURES));
|
||||
}
|
||||
} else {
|
||||
$this->consecutiveFailures = 0;
|
||||
}
|
||||
|
||||
if ($duration > self::SLOW_SECONDS) {
|
||||
$this->notBefore = $this->clock->now()->modify(sprintf('+%d ms', (int) (self::SLOW_PAUSE_SECONDS * 1000)));
|
||||
|
||||
if ($countSlow && ++$this->consecutiveSlow >= self::MAX_CONSECUTIVE_SLOW) {
|
||||
$this->consecutiveSlow = 0;
|
||||
throw new PagePhaseEndedException(sprintf('%d consecutive responses slower than %d s', self::MAX_CONSECUTIVE_SLOW, (int) self::SLOW_SECONDS));
|
||||
}
|
||||
} elseif ($countSlow) {
|
||||
$this->consecutiveSlow = 0;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function duration(ResponseInterface $response): float
|
||||
{
|
||||
$total = $response->getInfo('total_time');
|
||||
|
||||
return is_float($total) || is_int($total) ? (float) $total : 0.0;
|
||||
}
|
||||
|
||||
private function retryAfter(ResponseInterface $response): ?\DateTimeImmutable
|
||||
{
|
||||
$value = $response->getHeaders(false)['retry-after'][0] ?? null;
|
||||
|
||||
if (null === $value || '' === trim($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (ctype_digit(trim($value))) {
|
||||
return $this->clock->now()->modify(sprintf('+%d seconds', (int) trim($value)));
|
||||
}
|
||||
|
||||
$date = date_create_immutable($value);
|
||||
|
||||
return false === $date ? null : $date;
|
||||
}
|
||||
|
||||
private function waitUntil(\DateTimeImmutable $moment): void
|
||||
{
|
||||
$seconds = (float) $moment->format('U.u') - (float) $this->clock->now()->format('U.u');
|
||||
|
||||
if ($seconds > 0) {
|
||||
$this->clock->sleep($seconds);
|
||||
}
|
||||
}
|
||||
|
||||
private function apiUrl(string $path): string
|
||||
{
|
||||
return rtrim($this->baseUrl, '/').'/'.$path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use App\Chatbot\TripCatalog;
|
||||
use App\ContentFeed\Exception\ContentFeedStoppedException;
|
||||
use App\ContentFeed\Exception\ContentFeedUnavailableException;
|
||||
use App\ContentFeed\Exception\ContentRequestFailedException;
|
||||
use App\ContentFeed\Exception\PagePhaseEndedException;
|
||||
use App\Entity\ContentDocument;
|
||||
use App\Enum\ContentFetchState;
|
||||
use App\Model\ContentFeed\ContentManifest;
|
||||
use App\Model\ContentFeed\ContentSyncReport;
|
||||
use App\Model\ContentFeed\ManifestItem;
|
||||
use App\Repository\ContentDocumentRepository;
|
||||
use App\Repository\ContentSyncStateRepository;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
|
||||
/**
|
||||
* Mirrors the TYPO3 content feed into content_document.
|
||||
*
|
||||
* A run reads the manifest, removes what it no longer lists, refetches the records and
|
||||
* pages whose `modified` changed, and re-merges detail pages into their records. A failed
|
||||
* manifest changes nothing. Bulk page work belongs in the night window: outside it, a run
|
||||
* fetches at most PAGES_OUTSIDE_WINDOW pages and leaves the rest for the next one.
|
||||
*
|
||||
* With `refresh`, items whose stored copy is older than REFRESH_AFTER_DAYS are refetched
|
||||
* regardless of `modified`, oldest first: the backstop for the changes TYPO3 cannot see,
|
||||
* such as a referenced content element changing on another page.
|
||||
*/
|
||||
class ContentSyncer
|
||||
{
|
||||
public const PAGES_OUTSIDE_WINDOW = 50;
|
||||
public const REFRESH_AFTER_DAYS = 7;
|
||||
|
||||
private const NIGHT_WINDOW_TIMEZONE = 'Europe/Berlin';
|
||||
private const NIGHT_WINDOW_FROM_HOUR = 1;
|
||||
private const NIGHT_WINDOW_TO_HOUR = 5;
|
||||
private const FLUSH_EVERY_PAGES = 20;
|
||||
|
||||
public function __construct(
|
||||
private readonly ContentFeedClient $client,
|
||||
private readonly ContentDocumentRepository $documentRepository,
|
||||
private readonly ContentSyncStateRepository $stateRepository,
|
||||
private readonly DocumentBodyBuilder $bodyBuilder,
|
||||
private readonly HtmlToMarkdownConverter $converter,
|
||||
private readonly TripCatalog $tripCatalog,
|
||||
private readonly EntityManagerInterface $entityManager,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
}
|
||||
|
||||
public function run(bool $refresh = false, bool $ignoreWindow = false): ContentSyncReport
|
||||
{
|
||||
$report = new ContentSyncReport();
|
||||
$state = $this->stateRepository->get();
|
||||
$this->entityManager->persist($state);
|
||||
$now = $this->clock->now();
|
||||
|
||||
if (null !== $state->getPausedUntil() && $now < $state->getPausedUntil()) {
|
||||
$report->stoppedBy = 'Feed switched off by TYPO3 until '.$state->getPausedUntil()->format('c');
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
$state->setPausedUntil(null)->setLastRunAt($now);
|
||||
$this->client->startRun();
|
||||
|
||||
try {
|
||||
$manifest = $this->client->manifest();
|
||||
$documents = $this->applyManifest($manifest, $report);
|
||||
$state->setManifestGeneratedAt($manifest->generatedAt);
|
||||
$this->entityManager->flush();
|
||||
|
||||
$staleBefore = $refresh ? $now->modify(sprintf('-%d days', self::REFRESH_AFTER_DAYS)) : null;
|
||||
$this->syncRecords($documents, $staleBefore, $report);
|
||||
$this->syncPages($documents, $staleBefore, $ignoreWindow || $this->inNightWindow($now), $report);
|
||||
} catch (ContentFeedUnavailableException $e) {
|
||||
$state->setPausedUntil($e->retryAfter);
|
||||
$report->stoppedBy = $e->getMessage();
|
||||
} catch (ContentFeedStoppedException $e) {
|
||||
$report->stoppedBy = $e->getMessage();
|
||||
}
|
||||
|
||||
if (isset($documents)) {
|
||||
$this->merge();
|
||||
}
|
||||
|
||||
$state->setLastError($report->stoppedBy ?? $report->pagePhaseEndedBy);
|
||||
$this->entityManager->flush();
|
||||
|
||||
if (null === $report->stoppedBy) {
|
||||
$this->logger->info('Content feed sync finished', $report->toLogContext());
|
||||
} else {
|
||||
$this->logger->warning('Content feed sync stopped', $report->toLogContext());
|
||||
}
|
||||
|
||||
if ($report->isComplete() && $report->changedAnything()) {
|
||||
$this->tripCatalog->logUnmatchedCodes();
|
||||
}
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes what the manifest no longer lists and brings every listed item's metadata up
|
||||
* to date. Content is not touched here.
|
||||
*
|
||||
* @return array<string, array{document: ContentDocument, item: ManifestItem}>
|
||||
*/
|
||||
private function applyManifest(ContentManifest $manifest, ContentSyncReport $report): array
|
||||
{
|
||||
$stored = $this->documentRepository->findAllByKey();
|
||||
|
||||
foreach (array_diff_key($stored, $manifest->items) as $document) {
|
||||
$this->entityManager->remove($document);
|
||||
++$report->deleted;
|
||||
}
|
||||
|
||||
$documents = [];
|
||||
|
||||
foreach ($manifest->items as $key => $item) {
|
||||
$document = $stored[$key] ?? null;
|
||||
|
||||
if (null === $document) {
|
||||
$document = new ContentDocument($item->type, $item->uid, $item->language, $item->modified);
|
||||
$this->entityManager->persist($document);
|
||||
++$report->added;
|
||||
}
|
||||
|
||||
$document->setModified($item->modified)->setSite($item->site)->setUrl($item->url);
|
||||
|
||||
if ($document->isPage()) {
|
||||
$document
|
||||
->setTitle($item->title ?? '')
|
||||
->setBreadcrumb([] === $item->breadcrumb ? null : implode(' › ', $item->breadcrumb))
|
||||
->setData(['contentUrl' => $item->contentUrl, 'description' => $item->description, 'detailOf' => $item->detailOf]);
|
||||
}
|
||||
|
||||
$documents[$key] = ['document' => $document, 'item' => $item];
|
||||
}
|
||||
|
||||
return $documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*/
|
||||
private function syncRecords(array $documents, ?\DateTimeImmutable $staleBefore, ContentSyncReport $report): void
|
||||
{
|
||||
/** @var array<string, array<int, list<ContentDocument>>> $stale type => uid => documents; the records endpoint takes no language */
|
||||
$stale = [];
|
||||
|
||||
foreach ($this->dueForFetch($documents, $staleBefore, pages: false) as $document) {
|
||||
$stale[$document->getType()][$document->getUid()][] = $document;
|
||||
}
|
||||
|
||||
foreach ($stale as $type => $byUid) {
|
||||
foreach (array_chunk(array_keys($byUid), ContentFeedClient::RECORDS_BATCH_SIZE) as $uids) {
|
||||
try {
|
||||
$records = $this->client->records($type, $uids);
|
||||
} catch (ContentRequestFailedException $e) {
|
||||
$this->logger->warning('Content feed: records batch failed, retried next run', ['type' => $type, 'error' => $e->getMessage()]);
|
||||
$report->failed += count($uids);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($uids as $uid) {
|
||||
foreach ($byUid[$uid] as $document) {
|
||||
$record = $records[$uid] ?? null;
|
||||
|
||||
if (null === $record) {
|
||||
// Requested but absent: no longer eligible.
|
||||
$this->entityManager->remove($document);
|
||||
++$report->deleted;
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->applyRecord($document, $record);
|
||||
++$report->records;
|
||||
}
|
||||
}
|
||||
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
private function applyRecord(ContentDocument $document, array $record): void
|
||||
{
|
||||
$title = $this->bodyBuilder->title($document->getType(), $record);
|
||||
|
||||
$document
|
||||
->setData($record)
|
||||
->setTitle('' !== $title ? $title : $document->getTitle())
|
||||
// The merged pages are appended in merge().
|
||||
->setBodyMd($this->bodyBuilder->build($document->getType(), $record))
|
||||
->markFetched(ContentFetchState::Ok, $this->clock->now());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
private function syncPages(array $documents, ?\DateTimeImmutable $staleBefore, bool $unlimited, ContentSyncReport $report): void
|
||||
{
|
||||
$due = $this->dueForFetch($documents, $staleBefore, pages: true);
|
||||
|
||||
if (!$unlimited && count($due) > self::PAGES_OUTSIDE_WINDOW) {
|
||||
$report->deferred = count($due) - self::PAGES_OUTSIDE_WINDOW;
|
||||
$due = array_slice($due, 0, self::PAGES_OUTSIDE_WINDOW);
|
||||
}
|
||||
|
||||
$fetched = 0;
|
||||
|
||||
try {
|
||||
foreach ($due as $document) {
|
||||
$this->fetchPage($document, $report);
|
||||
|
||||
if (0 === ++$fetched % self::FLUSH_EVERY_PAGES) {
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
} catch (PagePhaseEndedException $e) {
|
||||
$report->pagePhaseEndedBy = $e->getMessage();
|
||||
$this->logger->warning('Content feed: page phase ended early', ['reason' => $e->getMessage()]);
|
||||
} finally {
|
||||
$this->entityManager->flush();
|
||||
}
|
||||
}
|
||||
|
||||
private function fetchPage(ContentDocument $document, ContentSyncReport $report): void
|
||||
{
|
||||
$contentUrl = $document->getData()['contentUrl'] ?? null;
|
||||
|
||||
if (!is_string($contentUrl)) {
|
||||
$document->markFetched(ContentFetchState::Failed, $this->clock->now());
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$result = $this->client->page($contentUrl);
|
||||
|
||||
if ($result->transient) {
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!$result->isOk()) {
|
||||
$this->logger->notice('Content feed: page failed, not refetched until it changes', [
|
||||
'uid' => $document->getUid(),
|
||||
'contentUrl' => $contentUrl,
|
||||
'status' => $result->failedStatus,
|
||||
]);
|
||||
$document->markFetched(ContentFetchState::Failed, $this->clock->now());
|
||||
++$report->failed;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$markdown = $this->converter->convert($result->html, $document->getUrl() ?? $contentUrl);
|
||||
$document->setBodyMd($markdown)->markFetched('' === $markdown ? ContentFetchState::Empty : ContentFetchState::Ok, $this->clock->now());
|
||||
++$report->pages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Items whose `modified` changed since their last fetch, plus, on a refresh, those
|
||||
* fetched before `$staleBefore`, oldest first.
|
||||
*
|
||||
* @param array<string, array{document: ContentDocument, item: ManifestItem}> $documents
|
||||
*
|
||||
* @return list<ContentDocument>
|
||||
*/
|
||||
private function dueForFetch(array $documents, ?\DateTimeImmutable $staleBefore, bool $pages): array
|
||||
{
|
||||
$due = [];
|
||||
|
||||
foreach ($documents as ['document' => $document]) {
|
||||
if ($document->isPage() !== $pages || !$this->entityManager->contains($document)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$outdated = null !== $staleBefore && (null === $document->getSyncedAt() || $document->getSyncedAt() < $staleBefore);
|
||||
|
||||
if ($document->isStale() || $outdated) {
|
||||
$due[] = $document;
|
||||
}
|
||||
}
|
||||
|
||||
usort($due, static fn (ContentDocument $a, ContentDocument $b): int => ($a->getSyncedAt()?->getTimestamp() ?? 0) <=> ($b->getSyncedAt()?->getTimestamp() ?? 0));
|
||||
|
||||
return $due;
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends every detail page to the record it is the detail page of, and keeps the page
|
||||
* itself out of the index. Recomputed from scratch on every run, so a change or deletion
|
||||
* on either side re-merges.
|
||||
*/
|
||||
private function merge(): void
|
||||
{
|
||||
$documents = $this->documentRepository->findAllByKey();
|
||||
/** @var array<string, list<ContentDocument>> $pagesByRecord */
|
||||
$pagesByRecord = [];
|
||||
|
||||
foreach ($documents as $page) {
|
||||
if (!$page->isPage()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$target = null;
|
||||
foreach ($page->getData()['detailOf'] ?? [] as $ref) {
|
||||
$candidate = $documents[ContentDocument::key($ref['type'], $ref['uid'], $page->getLanguage())] ?? null;
|
||||
|
||||
if (null !== $candidate && ContentFetchState::Ok === $candidate->getFetchState()) {
|
||||
$target = $candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$page->setMergedInto($target?->getRecordRef());
|
||||
|
||||
if (null !== $target && ContentFetchState::Ok === $page->getFetchState() && '' !== $page->getBodyMd()) {
|
||||
$pagesByRecord[$target->getKey()][] = $page;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($documents as $record) {
|
||||
if ($record->isPage() || ContentFetchState::Ok !== $record->getFetchState() || null === $record->getData()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$pages = $pagesByRecord[$record->getKey()] ?? [];
|
||||
usort($pages, static fn (ContentDocument $a, ContentDocument $b): int => $a->getUid() <=> $b->getUid());
|
||||
|
||||
$body = $this->bodyBuilder->build($record->getType(), $record->getData());
|
||||
foreach ($pages as $page) {
|
||||
$body .= "\n\n".$page->getBodyMd();
|
||||
}
|
||||
|
||||
$record->setBodyMd(trim($body));
|
||||
|
||||
// A record without a reachable page of its own is linkable through its detail page.
|
||||
if (null === $record->getUrl() && [] !== $pages) {
|
||||
$record->setUrl($pages[0]->getUrl())->setSite($record->getSite() ?? $pages[0]->getSite());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function inNightWindow(\DateTimeImmutable $now): bool
|
||||
{
|
||||
$hour = (int) $now->setTimezone(new \DateTimeZone(self::NIGHT_WINDOW_TIMEZONE))->format('G');
|
||||
|
||||
return $hour >= self::NIGHT_WINDOW_FROM_HOUR && $hour < self::NIGHT_WINDOW_TO_HOUR;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
/**
|
||||
* Builds the searchable Markdown body and the title of a feed record.
|
||||
*
|
||||
* Every record follows the same pattern: the name as heading, the short introductory
|
||||
* fields as paragraphs, then each `text` field as a section under a German heading, in a
|
||||
* fixed order. Structured facts that people ask about (WLAN, Sauna, Pistenkilometer) are
|
||||
* rendered as a list as well, so the full-text search finds them.
|
||||
*/
|
||||
class DocumentBodyBuilder
|
||||
{
|
||||
/**
|
||||
* Introductory fields, rendered without a heading. Plain text in the feed.
|
||||
*/
|
||||
private const LEAD_FIELDS = ['subline', 'headline'];
|
||||
|
||||
/**
|
||||
* `text` fields per type, in output order, with their section heading. A null heading
|
||||
* renders the field as an introduction straight under the title.
|
||||
*/
|
||||
private const TEXT_SECTIONS = [
|
||||
'product' => [
|
||||
'teaser' => null,
|
||||
'teaserLong' => 'Beschreibung',
|
||||
'feature' => 'Highlights',
|
||||
'conceptDescription' => 'Reisekonzept',
|
||||
'boardDescription' => 'Verpflegung',
|
||||
'includedServices' => 'Leistungen',
|
||||
'additionalServices' => 'Zusatzleistungen',
|
||||
'programmeDescription' => 'Programm',
|
||||
'skiPassDescription' => 'Skipass',
|
||||
'nonBookableInfo' => 'Hinweise zur Buchbarkeit',
|
||||
],
|
||||
'hotel' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'features' => 'Ausstattung',
|
||||
'roomTypes' => 'Zimmer',
|
||||
'additionalInformation' => 'Weitere Informationen',
|
||||
],
|
||||
'region' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'feature' => 'Highlights',
|
||||
'skiArea' => 'Skigebiet',
|
||||
'skiAreaExtended' => 'Skigebiet im Detail',
|
||||
'skiPass' => 'Skipass',
|
||||
'news' => 'Neuigkeiten',
|
||||
],
|
||||
'country' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
],
|
||||
'city' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
'party' => 'Party',
|
||||
'leisure' => 'Freizeit',
|
||||
],
|
||||
'concept' => [
|
||||
'teaser' => null,
|
||||
'description' => 'Beschreibung',
|
||||
'descriptionExtended' => 'Mehr erfahren',
|
||||
'popupText' => 'Kurz erklärt',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Facts per type, with their German label.
|
||||
*/
|
||||
private const FACTS = [
|
||||
'hotel' => [
|
||||
'bedsCount' => 'Betten',
|
||||
'parkingCount' => 'Parkplätze',
|
||||
'altitude' => 'Höhenlage',
|
||||
'distanceBus' => 'Entfernung zur Bushaltestelle',
|
||||
'distanceLift' => 'Entfernung zum Lift',
|
||||
'shower' => 'Dusche/WC',
|
||||
'partyroom' => 'Partyraum',
|
||||
'gamesroom' => 'Spieleraum',
|
||||
'sauna' => 'Sauna',
|
||||
'wifi' => 'WLAN',
|
||||
],
|
||||
'region' => [
|
||||
'length' => 'Pistenkilometer',
|
||||
'height' => 'Höhe',
|
||||
'lifts' => 'Lifte',
|
||||
],
|
||||
];
|
||||
|
||||
public function __construct(private readonly HtmlToMarkdownConverter $converter)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
public function title(string $type, array $record): string
|
||||
{
|
||||
$title = 'news' === $type ? ($record['title'] ?? null) : ($record['name'] ?? null);
|
||||
|
||||
return is_string($title) ? trim($title) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*/
|
||||
public function build(string $type, array $record): string
|
||||
{
|
||||
$baseUrl = is_string($record['url'] ?? null) ? $record['url'] : null;
|
||||
$blocks = [];
|
||||
|
||||
$title = $this->title($type, $record);
|
||||
if ('' !== $title) {
|
||||
$blocks[] = '# '.$title;
|
||||
}
|
||||
|
||||
if ('news' === $type) {
|
||||
return $this->join(array_merge($blocks, $this->newsBlocks($record, $baseUrl)));
|
||||
}
|
||||
|
||||
foreach (self::LEAD_FIELDS as $field) {
|
||||
$value = $this->plain($record[$field] ?? null);
|
||||
if ('' !== $value && $value !== $title) {
|
||||
$blocks[] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
if ('hotel' === $type) {
|
||||
$address = $this->plain($record['address'] ?? null);
|
||||
if ('' !== $address) {
|
||||
$blocks[] = 'Adresse: '.$address;
|
||||
}
|
||||
}
|
||||
|
||||
$text = is_array($record['text'] ?? null) ? $record['text'] : [];
|
||||
foreach (self::TEXT_SECTIONS[$type] ?? [] as $field => $heading) {
|
||||
$markdown = $this->converter->convert(is_string($text[$field] ?? null) ? $text[$field] : null, $baseUrl);
|
||||
if ('' === $markdown) {
|
||||
continue;
|
||||
}
|
||||
$blocks[] = null === $heading ? $markdown : "## {$heading}\n\n{$markdown}";
|
||||
}
|
||||
|
||||
$facts = $this->facts($type, is_array($record['facts'] ?? null) ? $record['facts'] : []);
|
||||
if ('' !== $facts) {
|
||||
$blocks[] = "## Fakten\n\n".$facts;
|
||||
}
|
||||
|
||||
if ('region' === $type && is_string($record['webcam'] ?? null) && '' !== trim($record['webcam'])) {
|
||||
$blocks[] = 'Webcam: '.trim($record['webcam']);
|
||||
}
|
||||
|
||||
if ('product' === $type) {
|
||||
$faqs = $this->faqs(is_array($record['faqs'] ?? null) ? $record['faqs'] : [], $baseUrl);
|
||||
if ('' !== $faqs) {
|
||||
$blocks[] = "## Häufige Fragen\n\n".$faqs;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->join($blocks);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $record
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function newsBlocks(array $record, ?string $baseUrl): array
|
||||
{
|
||||
$blocks = [];
|
||||
|
||||
if (is_string($record['datetime'] ?? null) && false !== ($date = date_create_immutable($record['datetime']))) {
|
||||
$blocks[] = 'Veröffentlicht am '.$date->format('d.m.Y');
|
||||
}
|
||||
|
||||
foreach (['teaser', 'bodytext'] as $field) {
|
||||
$markdown = $this->converter->convert(is_string($record[$field] ?? null) ? $record[$field] : null, $baseUrl);
|
||||
if ('' !== $markdown) {
|
||||
$blocks[] = $markdown;
|
||||
}
|
||||
}
|
||||
|
||||
$tags = $this->labels($record['tags'] ?? null);
|
||||
if ([] !== $tags) {
|
||||
$blocks[] = 'Themen: '.implode(', ', $tags);
|
||||
}
|
||||
|
||||
return $blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $facts
|
||||
*/
|
||||
private function facts(string $type, array $facts): string
|
||||
{
|
||||
$lines = [];
|
||||
|
||||
foreach (self::FACTS[$type] ?? [] as $field => $label) {
|
||||
$value = $facts[$field] ?? null;
|
||||
|
||||
$rendered = match (true) {
|
||||
is_bool($value) => $value ? 'ja' : 'nein',
|
||||
is_int($value), is_float($value) => (string) $value,
|
||||
is_string($value) => $this->plain($value),
|
||||
default => '',
|
||||
};
|
||||
|
||||
if ('' !== $rendered) {
|
||||
$lines[] = "- {$label}: {$rendered}";
|
||||
}
|
||||
}
|
||||
|
||||
return implode("\n", $lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<mixed> $faqs
|
||||
*/
|
||||
private function faqs(array $faqs, ?string $baseUrl): string
|
||||
{
|
||||
$blocks = [];
|
||||
|
||||
foreach ($faqs as $faq) {
|
||||
if (!is_array($faq)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$question = $this->plain($faq['question'] ?? null);
|
||||
$answer = $this->converter->convert(is_string($faq['answer'] ?? null) ? $faq['answer'] : null, $baseUrl);
|
||||
|
||||
if ('' !== $question && '' !== $answer) {
|
||||
$blocks[] = "### {$question}\n\n{$answer}";
|
||||
}
|
||||
}
|
||||
|
||||
return $this->join($blocks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tags and categories may arrive as strings or as objects with a title or name.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function labels(mixed $values): array
|
||||
{
|
||||
if (!is_array($values)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$labels = [];
|
||||
foreach ($values as $value) {
|
||||
$label = is_array($value) ? ($value['title'] ?? $value['name'] ?? null) : $value;
|
||||
if (is_string($label) && '' !== trim($label)) {
|
||||
$labels[] = trim($label);
|
||||
}
|
||||
}
|
||||
|
||||
return $labels;
|
||||
}
|
||||
|
||||
/**
|
||||
* A field that is plain text in the feed, defensively stripped of any markup.
|
||||
*/
|
||||
private function plain(mixed $value): string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return trim((string) preg_replace('/\s+/u', ' ', html_entity_decode(strip_tags($value), \ENT_QUOTES | \ENT_HTML5, 'UTF-8')));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<string> $blocks
|
||||
*/
|
||||
private function join(array $blocks): string
|
||||
{
|
||||
return implode("\n\n", $blocks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* Ends the whole sync run, keeping everything as it is. The next scheduled run tries again.
|
||||
*
|
||||
* Thrown for a manifest that cannot be trusted, three consecutive 5xx responses or
|
||||
* timeouts, and anything else that means TYPO3 is not in a state to be read.
|
||||
*/
|
||||
class ContentFeedStoppedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* TYPO3 answered 503: the feed has been switched off on its side.
|
||||
*/
|
||||
class ContentFeedUnavailableException extends ContentFeedStoppedException
|
||||
{
|
||||
public function __construct(public readonly ?\DateTimeImmutable $retryAfter)
|
||||
{
|
||||
parent::__construct('TYPO3 switched the content feed off (503)'.(null !== $retryAfter ? ', retry after '.$retryAfter->format('c') : ''));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* One request failed without ending the run. The items it was for keep their stored
|
||||
* version and are retried next run.
|
||||
*/
|
||||
class ContentRequestFailedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed\Exception;
|
||||
|
||||
/**
|
||||
* Three slow responses in a row: TYPO3 is under load, so no more pages this run. The rest
|
||||
* of the run (bookkeeping, merging) still completes.
|
||||
*/
|
||||
class PagePhaseEndedException extends \RuntimeException
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use League\HTMLToMarkdown\Converter\TableConverter;
|
||||
use League\HTMLToMarkdown\HtmlConverter;
|
||||
use Symfony\Component\DomCrawler\UriResolver;
|
||||
|
||||
/**
|
||||
* Turns the CMS's HTML, record text fields and page fragments alike, into Markdown for the
|
||||
* index and the tool output. LLMs read Markdown well; HTML noise only costs tokens.
|
||||
*
|
||||
* Embeds, media and forms are dropped with their content. Relative links are made
|
||||
* absolute against the document's own URL, since the page fragment may carry them.
|
||||
*/
|
||||
class HtmlToMarkdownConverter
|
||||
{
|
||||
private const REMOVED_ELEMENTS = ['script', 'style', 'noscript', 'iframe', 'img', 'picture', 'svg', 'form'];
|
||||
|
||||
private readonly HtmlConverter $converter;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->converter = new HtmlConverter([
|
||||
'header_style' => 'atx',
|
||||
'strip_tags' => true,
|
||||
'strip_placeholder_links' => true,
|
||||
'hard_break' => true,
|
||||
'suppress_errors' => true,
|
||||
'remove_nodes' => implode(' ', self::REMOVED_ELEMENTS),
|
||||
]);
|
||||
$this->converter->getEnvironment()->addConverter(new TableConverter());
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $baseUrl what relative links resolve against; left as they are without one
|
||||
*/
|
||||
public function convert(?string $html, ?string $baseUrl = null): string
|
||||
{
|
||||
if (null === $html || '' === trim($html)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$markdown = $this->converter->convert($this->prepare($html, $baseUrl));
|
||||
|
||||
return $this->collapseWhitespace($markdown);
|
||||
}
|
||||
|
||||
private function prepare(string $html, ?string $baseUrl): string
|
||||
{
|
||||
$document = \Dom\HTMLDocument::createFromString('<!DOCTYPE html><html><body>'.$html.'</body></html>', \LIBXML_NOERROR, 'UTF-8');
|
||||
$body = $document->body;
|
||||
|
||||
if (null === $body) {
|
||||
return $html;
|
||||
}
|
||||
|
||||
foreach ($body->querySelectorAll(implode(',', self::REMOVED_ELEMENTS)) as $element) {
|
||||
$element->remove();
|
||||
}
|
||||
|
||||
// Whitespace in text collapses as a browser would; the template's indentation would
|
||||
// otherwise end up inside link labels and headings.
|
||||
foreach ((new \Dom\XPath($document))->query('//body//text()[not(ancestor::pre)]') as $text) {
|
||||
if (!$text instanceof \Dom\Text) {
|
||||
continue;
|
||||
}
|
||||
$text->data = (string) preg_replace('/[ \t\r\n]+/', ' ', $text->data);
|
||||
}
|
||||
|
||||
if (null !== $baseUrl) {
|
||||
foreach ($body->querySelectorAll('a[href]') as $link) {
|
||||
$href = trim((string) $link->getAttribute('href'));
|
||||
|
||||
if ('' === $href || preg_match('/^(mailto|tel|javascript):/i', $href)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$link->setAttribute('href', UriResolver::resolve($href, $baseUrl));
|
||||
}
|
||||
}
|
||||
|
||||
return $body->innerHTML;
|
||||
}
|
||||
|
||||
private function collapseWhitespace(string $markdown): string
|
||||
{
|
||||
// The converter leaves text entity-encoded (`&`), which only costs tokens.
|
||||
$markdown = html_entity_decode($markdown, \ENT_QUOTES | \ENT_HTML5, 'UTF-8');
|
||||
$markdown = str_replace(["\r\n", "\r", "\u{00A0}"], ["\n", "\n", ' '], $markdown);
|
||||
|
||||
// Labels padded by the collapsed whitespace: "[ Kontakt ](…)" -> "[Kontakt](…)".
|
||||
$markdown = (string) preg_replace(['/\[ +/', '/ +\]\(/'], ['[', ']('], $markdown);
|
||||
|
||||
$lines = array_map(
|
||||
// Inner runs only: leading spaces carry list nesting.
|
||||
static fn (string $line): string => rtrim((string) preg_replace('/(?<=\S)[ \t]{2,}/', ' ', $line)),
|
||||
explode("\n", $markdown),
|
||||
);
|
||||
|
||||
return trim((string) preg_replace("/\n{3,}/", "\n\n", implode("\n", $lines)));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user