*/ private array $siteHosts = []; public function __construct( private readonly HttpClientInterface $httpClient, private readonly ClockInterface $clock, private readonly LoggerInterface $logger, private readonly string $baseUrl, private readonly string $apiKey, ) { } public function startRun(): void { $this->consecutiveFailures = 0; $this->consecutiveSlow = 0; $this->notBefore = null; $this->lastPageAt = null; $this->siteHosts = []; } /** * The whole manifest. Anything but a usable one stops the run: an empty TYPO3 is a bug, * not a deletion. * * @throws ContentFeedStoppedException */ public function manifest(): ContentManifest { if ('' === $this->apiKey) { throw new ContentFeedStoppedException('APP_CMS_API_KEY is not configured'); } $response = $this->send($this->apiUrl('api/content/manifest'), []); if (null === $response || 200 !== $response['status']) { throw new ContentFeedStoppedException(sprintf('Manifest request failed (%s)', $response['status'] ?? 'timeout')); } try { $data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR); } catch (\JsonException $e) { throw new ContentFeedStoppedException('Manifest is not valid JSON', 0, $e); } if (!is_array($data) || !is_string($data['generatedAt'] ?? null) || !is_array($data['items'] ?? null) || [] === $data['items']) { throw new ContentFeedStoppedException('Manifest has no generatedAt or no items'); } $generatedAt = ManifestItem::localTime($data['generatedAt']); if (null === $generatedAt) { throw new ContentFeedStoppedException('Manifest generatedAt is unparseable'); } $items = []; foreach ($data['items'] as $raw) { try { $item = ManifestItem::fromArray(is_array($raw) ? $raw : []); } catch (\InvalidArgumentException $e) { // A malformed entry would otherwise read as a deletion of whatever it was. throw new ContentFeedStoppedException('Manifest contains an unusable item: '.$e->getMessage(), 0, $e); } $items[$item->key()] = $item; } $hosts = []; foreach (is_array($data['sites'] ?? null) ? $data['sites'] : [] as $site) { $host = is_array($site) && is_string($site['base'] ?? null) ? parse_url($site['base'], \PHP_URL_HOST) : null; if (is_string($host)) { $hosts[] = strtolower($host); } } $this->siteHosts = array_values(array_unique($hosts)); return new ContentManifest($generatedAt, $items, $this->siteHosts); } /** * Records of one type by uid, keyed by uid. A requested uid that is absent is no * longer eligible. * * @param list $uids at most RECORDS_BATCH_SIZE * * @return array> * * @throws ContentRequestFailedException this batch failed, the run goes on * @throws ContentFeedStoppedException */ public function records(string $type, array $uids): array { if (count($uids) > self::RECORDS_BATCH_SIZE) { throw new \InvalidArgumentException(sprintf('At most %d uids per records request', self::RECORDS_BATCH_SIZE)); } $response = $this->send($this->apiUrl('api/content/records'), ['type' => $type, 'uids' => implode(',', $uids)]); if (null === $response) { throw new ContentRequestFailedException(sprintf('Records request for %s timed out', $type)); } if (200 !== $response['status']) { if ($response['status'] >= 500) { throw new ContentRequestFailedException(sprintf('Records request for %s failed (%d)', $type, $response['status'])); } // 403 and the like: misconfiguration, which the next request would hit too. throw new ContentFeedStoppedException(sprintf('Records request for %s answered %d', $type, $response['status'])); } try { $data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR); } catch (\JsonException $e) { throw new ContentRequestFailedException(sprintf('Records response for %s is not valid JSON', $type), 0, $e); } // Anything but the documented envelope fails the batch: read as an empty list, it // would delete every record requested. if (!is_array($data) || ($data['type'] ?? null) !== $type || !is_array($data['items'] ?? null) || !array_is_list($data['items'])) { throw new ContentRequestFailedException(sprintf('Records response for %s has no items list', $type)); } $records = []; foreach ($data['items'] as $record) { if (is_array($record) && is_int($record['uid'] ?? null)) { $records[$record['uid']] = $record; } } return $records; } /** * The bare HTML fragment of one page, fetched at most once a second. * * @throws PagePhaseEndedException * @throws ContentFeedStoppedException */ public function page(string $contentUrl): PageFetchResult { $host = parse_url($contentUrl, \PHP_URL_HOST); // The key goes with every request, so it only ever goes to a host of the feed's own sites. if (!is_string($host) || !in_array(strtolower($host), $this->siteHosts, true)) { $this->logger->warning('Content feed: contentUrl outside the manifest sites, skipped', ['contentUrl' => $contentUrl]); return PageFetchResult::failed(0); } if (null !== $this->lastPageAt) { $this->waitUntil($this->lastPageAt->modify(sprintf('+%d ms', (int) (self::PAGE_INTERVAL_SECONDS * 1000)))); } $this->lastPageAt = $this->clock->now(); $response = $this->send($contentUrl, [], countSlow: true); if (null === $response) { return PageFetchResult::transient(); } if (200 === $response['status']) { return PageFetchResult::ok($response['body']); } return PageFetchResult::failed($response['status']); } /** * Sends one GET under the load rules. Null on a timeout or transport error. * * @param array $query * * @return array{status: int, body: string}|null * * @throws ContentFeedStoppedException * @throws PagePhaseEndedException */ private function send(string $url, array $query, bool $countSlow = false): ?array { if (null !== $this->notBefore) { $this->waitUntil($this->notBefore); $this->notBefore = null; } $result = null; $duration = 0.0; try { $response = $this->httpClient->request('GET', $url, [ 'query' => $query, 'headers' => ['X-Api-Key' => $this->apiKey], 'timeout' => self::TIMEOUT_SECONDS, 'max_duration' => self::TIMEOUT_SECONDS, ]); $result = ['status' => $response->getStatusCode(), 'body' => $response->getContent(false)]; $duration = $this->duration($response); } catch (TransportExceptionInterface $e) { $this->logger->warning('Content feed: request failed', ['url' => $url, 'error' => $e->getMessage()]); $duration = self::TIMEOUT_SECONDS; } if (null !== $result && 503 === $result['status']) { throw new ContentFeedUnavailableException($this->retryAfter($response)); } if (null === $result || $result['status'] >= 500) { if (++$this->consecutiveFailures >= self::MAX_CONSECUTIVE_FAILURES) { throw new ContentFeedStoppedException(sprintf('%d consecutive 5xx responses or timeouts', self::MAX_CONSECUTIVE_FAILURES)); } } else { $this->consecutiveFailures = 0; } if ($duration > self::SLOW_SECONDS) { $this->notBefore = $this->clock->now()->modify(sprintf('+%d ms', (int) (self::SLOW_PAUSE_SECONDS * 1000))); if ($countSlow && ++$this->consecutiveSlow >= self::MAX_CONSECUTIVE_SLOW) { $this->consecutiveSlow = 0; throw new PagePhaseEndedException(sprintf('%d consecutive responses slower than %d s', self::MAX_CONSECUTIVE_SLOW, (int) self::SLOW_SECONDS)); } } elseif ($countSlow) { $this->consecutiveSlow = 0; } return $result; } private function duration(ResponseInterface $response): float { $total = $response->getInfo('total_time'); return is_float($total) || is_int($total) ? (float) $total : 0.0; } private function retryAfter(ResponseInterface $response): ?\DateTimeImmutable { $value = $response->getHeaders(false)['retry-after'][0] ?? null; if (null === $value || '' === trim($value)) { return null; } if (ctype_digit(trim($value))) { return $this->clock->now()->modify(sprintf('+%d seconds', (int) trim($value))); } $date = date_create_immutable($value); return false === $date ? null : $date; } private function waitUntil(\DateTimeImmutable $moment): void { $seconds = (float) $moment->format('U.u') - (float) $this->clock->now()->format('U.u'); if ($seconds > 0) { $this->clock->sleep($seconds); } } private function apiUrl(string $path): string { return rtrim($this->baseUrl, '/').'/'.$path; } }