feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed
This commit is contained in:
fromme committed
1 parent
ff7ddf7a9e
commit
1b4384e3f3
52 files changed
+5117
-4
No files matched your search
@@ -0,0 +1,312 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\ContentFeed;
|
||||
|
||||
use App\ContentFeed\Exception\ContentFeedStoppedException;
|
||||
use App\ContentFeed\Exception\ContentFeedUnavailableException;
|
||||
use App\ContentFeed\Exception\ContentRequestFailedException;
|
||||
use App\ContentFeed\Exception\PagePhaseEndedException;
|
||||
use App\Model\ContentFeed\ContentManifest;
|
||||
use App\Model\ContentFeed\ManifestItem;
|
||||
use App\Model\ContentFeed\PageFetchResult;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Clock\ClockInterface;
|
||||
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
|
||||
use Symfony\Contracts\HttpClient\HttpClientInterface;
|
||||
use Symfony\Contracts\HttpClient\ResponseInterface;
|
||||
|
||||
/**
|
||||
* Reads the TYPO3 content feed: the manifest, records by uid, and rendered page fragments.
|
||||
*
|
||||
* The website is an old TYPO3 install on which every page fetch is a full frontend render,
|
||||
* so the load rules here are a contract with its maintainers, not tuning:
|
||||
*
|
||||
* - one request at a time, never parallel;
|
||||
* - at most one page fetch per second;
|
||||
* - after a response slower than 2 s, wait 5 s; after three in a row, no more pages this run;
|
||||
* - a 503 (the kill switch) or three consecutive 5xx responses or timeouts end the run.
|
||||
*
|
||||
* One instance serves one run: call startRun() first, it resets the counters.
|
||||
*/
|
||||
class ContentFeedClient
|
||||
{
|
||||
public const RECORDS_BATCH_SIZE = 100;
|
||||
|
||||
private const TIMEOUT_SECONDS = 10.0;
|
||||
private const SLOW_SECONDS = 2.0;
|
||||
private const SLOW_PAUSE_SECONDS = 5.0;
|
||||
private const PAGE_INTERVAL_SECONDS = 1.0;
|
||||
private const MAX_CONSECUTIVE_FAILURES = 3;
|
||||
private const MAX_CONSECUTIVE_SLOW = 3;
|
||||
|
||||
private int $consecutiveFailures = 0;
|
||||
private int $consecutiveSlow = 0;
|
||||
private ?\DateTimeImmutable $notBefore = null;
|
||||
private ?\DateTimeImmutable $lastPageAt = null;
|
||||
|
||||
/** @var list<string> */
|
||||
private array $siteHosts = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly HttpClientInterface $httpClient,
|
||||
private readonly ClockInterface $clock,
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly string $baseUrl,
|
||||
private readonly string $apiKey,
|
||||
) {
|
||||
}
|
||||
|
||||
public function startRun(): void
|
||||
{
|
||||
$this->consecutiveFailures = 0;
|
||||
$this->consecutiveSlow = 0;
|
||||
$this->notBefore = null;
|
||||
$this->lastPageAt = null;
|
||||
$this->siteHosts = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole manifest. Anything but a usable one stops the run: an empty TYPO3 is a bug,
|
||||
* not a deletion.
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function manifest(): ContentManifest
|
||||
{
|
||||
if ('' === $this->apiKey) {
|
||||
throw new ContentFeedStoppedException('APP_CMS_API_KEY is not configured');
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/manifest'), []);
|
||||
|
||||
if (null === $response || 200 !== $response['status']) {
|
||||
throw new ContentFeedStoppedException(sprintf('Manifest request failed (%s)', $response['status'] ?? 'timeout'));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentFeedStoppedException('Manifest is not valid JSON', 0, $e);
|
||||
}
|
||||
|
||||
if (!is_array($data) || !is_string($data['generatedAt'] ?? null) || !is_array($data['items'] ?? null) || [] === $data['items']) {
|
||||
throw new ContentFeedStoppedException('Manifest has no generatedAt or no items');
|
||||
}
|
||||
|
||||
$generatedAt = ManifestItem::localTime($data['generatedAt']);
|
||||
if (null === $generatedAt) {
|
||||
throw new ContentFeedStoppedException('Manifest generatedAt is unparseable');
|
||||
}
|
||||
|
||||
$items = [];
|
||||
foreach ($data['items'] as $raw) {
|
||||
try {
|
||||
$item = ManifestItem::fromArray(is_array($raw) ? $raw : []);
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
// A malformed entry would otherwise read as a deletion of whatever it was.
|
||||
throw new ContentFeedStoppedException('Manifest contains an unusable item: '.$e->getMessage(), 0, $e);
|
||||
}
|
||||
$items[$item->key()] = $item;
|
||||
}
|
||||
|
||||
$hosts = [];
|
||||
foreach (is_array($data['sites'] ?? null) ? $data['sites'] : [] as $site) {
|
||||
$host = is_array($site) && is_string($site['base'] ?? null) ? parse_url($site['base'], \PHP_URL_HOST) : null;
|
||||
if (is_string($host)) {
|
||||
$hosts[] = strtolower($host);
|
||||
}
|
||||
}
|
||||
$this->siteHosts = array_values(array_unique($hosts));
|
||||
|
||||
return new ContentManifest($generatedAt, $items, $this->siteHosts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Records of one type by uid, keyed by uid. A requested uid that is absent is no
|
||||
* longer eligible.
|
||||
*
|
||||
* @param list<int> $uids at most RECORDS_BATCH_SIZE
|
||||
*
|
||||
* @return array<int, array<string, mixed>>
|
||||
*
|
||||
* @throws ContentRequestFailedException this batch failed, the run goes on
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function records(string $type, array $uids): array
|
||||
{
|
||||
if (count($uids) > self::RECORDS_BATCH_SIZE) {
|
||||
throw new \InvalidArgumentException(sprintf('At most %d uids per records request', self::RECORDS_BATCH_SIZE));
|
||||
}
|
||||
|
||||
$response = $this->send($this->apiUrl('api/content/records'), ['type' => $type, 'uids' => implode(',', $uids)]);
|
||||
|
||||
if (null === $response) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s timed out', $type));
|
||||
}
|
||||
|
||||
if (200 !== $response['status']) {
|
||||
if ($response['status'] >= 500) {
|
||||
throw new ContentRequestFailedException(sprintf('Records request for %s failed (%d)', $type, $response['status']));
|
||||
}
|
||||
|
||||
// 403 and the like: misconfiguration, which the next request would hit too.
|
||||
throw new ContentFeedStoppedException(sprintf('Records request for %s answered %d', $type, $response['status']));
|
||||
}
|
||||
|
||||
try {
|
||||
$data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s is not valid JSON', $type), 0, $e);
|
||||
}
|
||||
|
||||
// Anything but the documented envelope fails the batch: read as an empty list, it
|
||||
// would delete every record requested.
|
||||
if (!is_array($data) || ($data['type'] ?? null) !== $type || !is_array($data['items'] ?? null) || !array_is_list($data['items'])) {
|
||||
throw new ContentRequestFailedException(sprintf('Records response for %s has no items list', $type));
|
||||
}
|
||||
|
||||
$records = [];
|
||||
foreach ($data['items'] as $record) {
|
||||
if (is_array($record) && is_int($record['uid'] ?? null)) {
|
||||
$records[$record['uid']] = $record;
|
||||
}
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
/**
|
||||
* The bare HTML fragment of one page, fetched at most once a second.
|
||||
*
|
||||
* @throws PagePhaseEndedException
|
||||
* @throws ContentFeedStoppedException
|
||||
*/
|
||||
public function page(string $contentUrl): PageFetchResult
|
||||
{
|
||||
$host = parse_url($contentUrl, \PHP_URL_HOST);
|
||||
|
||||
// The key goes with every request, so it only ever goes to a host of the feed's own sites.
|
||||
if (!is_string($host) || !in_array(strtolower($host), $this->siteHosts, true)) {
|
||||
$this->logger->warning('Content feed: contentUrl outside the manifest sites, skipped', ['contentUrl' => $contentUrl]);
|
||||
|
||||
return PageFetchResult::failed(0);
|
||||
}
|
||||
|
||||
if (null !== $this->lastPageAt) {
|
||||
$this->waitUntil($this->lastPageAt->modify(sprintf('+%d ms', (int) (self::PAGE_INTERVAL_SECONDS * 1000))));
|
||||
}
|
||||
$this->lastPageAt = $this->clock->now();
|
||||
|
||||
$response = $this->send($contentUrl, [], countSlow: true);
|
||||
|
||||
if (null === $response) {
|
||||
return PageFetchResult::transient();
|
||||
}
|
||||
|
||||
if (200 === $response['status']) {
|
||||
return PageFetchResult::ok($response['body']);
|
||||
}
|
||||
|
||||
return PageFetchResult::failed($response['status']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends one GET under the load rules. Null on a timeout or transport error.
|
||||
*
|
||||
* @param array<string, string> $query
|
||||
*
|
||||
* @return array{status: int, body: string}|null
|
||||
*
|
||||
* @throws ContentFeedStoppedException
|
||||
* @throws PagePhaseEndedException
|
||||
*/
|
||||
private function send(string $url, array $query, bool $countSlow = false): ?array
|
||||
{
|
||||
if (null !== $this->notBefore) {
|
||||
$this->waitUntil($this->notBefore);
|
||||
$this->notBefore = null;
|
||||
}
|
||||
|
||||
$result = null;
|
||||
$duration = 0.0;
|
||||
|
||||
try {
|
||||
$response = $this->httpClient->request('GET', $url, [
|
||||
'query' => $query,
|
||||
'headers' => ['X-Api-Key' => $this->apiKey],
|
||||
'timeout' => self::TIMEOUT_SECONDS,
|
||||
'max_duration' => self::TIMEOUT_SECONDS,
|
||||
]);
|
||||
$result = ['status' => $response->getStatusCode(), 'body' => $response->getContent(false)];
|
||||
$duration = $this->duration($response);
|
||||
} catch (TransportExceptionInterface $e) {
|
||||
$this->logger->warning('Content feed: request failed', ['url' => $url, 'error' => $e->getMessage()]);
|
||||
$duration = self::TIMEOUT_SECONDS;
|
||||
}
|
||||
|
||||
if (null !== $result && 503 === $result['status']) {
|
||||
throw new ContentFeedUnavailableException($this->retryAfter($response));
|
||||
}
|
||||
|
||||
if (null === $result || $result['status'] >= 500) {
|
||||
if (++$this->consecutiveFailures >= self::MAX_CONSECUTIVE_FAILURES) {
|
||||
throw new ContentFeedStoppedException(sprintf('%d consecutive 5xx responses or timeouts', self::MAX_CONSECUTIVE_FAILURES));
|
||||
}
|
||||
} else {
|
||||
$this->consecutiveFailures = 0;
|
||||
}
|
||||
|
||||
if ($duration > self::SLOW_SECONDS) {
|
||||
$this->notBefore = $this->clock->now()->modify(sprintf('+%d ms', (int) (self::SLOW_PAUSE_SECONDS * 1000)));
|
||||
|
||||
if ($countSlow && ++$this->consecutiveSlow >= self::MAX_CONSECUTIVE_SLOW) {
|
||||
$this->consecutiveSlow = 0;
|
||||
throw new PagePhaseEndedException(sprintf('%d consecutive responses slower than %d s', self::MAX_CONSECUTIVE_SLOW, (int) self::SLOW_SECONDS));
|
||||
}
|
||||
} elseif ($countSlow) {
|
||||
$this->consecutiveSlow = 0;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function duration(ResponseInterface $response): float
|
||||
{
|
||||
$total = $response->getInfo('total_time');
|
||||
|
||||
return is_float($total) || is_int($total) ? (float) $total : 0.0;
|
||||
}
|
||||
|
||||
private function retryAfter(ResponseInterface $response): ?\DateTimeImmutable
|
||||
{
|
||||
$value = $response->getHeaders(false)['retry-after'][0] ?? null;
|
||||
|
||||
if (null === $value || '' === trim($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (ctype_digit(trim($value))) {
|
||||
return $this->clock->now()->modify(sprintf('+%d seconds', (int) trim($value)));
|
||||
}
|
||||
|
||||
$date = date_create_immutable($value);
|
||||
|
||||
return false === $date ? null : $date;
|
||||
}
|
||||
|
||||
private function waitUntil(\DateTimeImmutable $moment): void
|
||||
{
|
||||
$seconds = (float) $moment->format('U.u') - (float) $this->clock->now()->format('U.u');
|
||||
|
||||
if ($seconds > 0) {
|
||||
$this->clock->sleep($seconds);
|
||||
}
|
||||
}
|
||||
|
||||
private function apiUrl(string $path): string
|
||||
{
|
||||
return rtrim($this->baseUrl, '/').'/'.$path;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user