feat: add the key-protected content-only page type for the chatbot feed

public/typo3conf/ext/ep_products/Classes/ContentFeed/ManifestBuilder.php

@@ -111,7 +111,7 @@ class ManifestBuilder
      */
     private function buildPageItems(array $sites, array $detailPages): array
     {
-        $contentTimestamps = $this->findContentTimestamps();
+        $content = $this->findContentStatistics();
         $excluded = array_flip($this->configuration->getExcludedPages());
         $feedLanguages = $this->configuration->getLanguages();
         $items = [];
@@ -129,7 +129,7 @@ class ManifestBuilder
                 'site' => $site,
                 'languages' => $languageIds,
                 'excluded' => $excluded,
-                'contentTimestamps' => $contentTimestamps,
+                'content' => $content,
                 'detailPages' => $detailPages,
             ];
             $breadcrumbs = array_fill_keys(array_keys($languageIds), []);
@@ -214,6 +214,25 @@ class ManifestBuilder
     private function buildPageItem(array $page, array $row, int $languageId, string $isoCode, array $breadcrumb, array $context): ?array
     {
         $uid = (int) $page['uid'];
+        // "Show content from page": the elements live on another page, and so do their changes.
+        // The site answers 404 when that page is hidden.
+        $contentPid = (int) $page['content_from_pid'] ?: $uid;
+        $contentPage = $this->pageTree->getPage($contentPid);
+
+        if (null === $contentPage || !$this->pageTree->isVisible($contentPage)) {
+            return null;
+        }
+
+        // Without a visible element other than plugins, the content-only render is empty (it
+        // leaves plugins out), so there is nothing to fetch. That is most product and hotel
+        // detail pages: their text comes through the records endpoint.
+        $content = $context['content'][$contentPid] ?? [];
+        $textElements = ($content[$languageId]['text'] ?? 0) + ($content[-1]['text'] ?? 0);
+
+        if (0 === $textElements) {
+            return null;
+        }
+
         $url = $this->urlResolver->getPageUrl($uid, $languageId);
 
         if (null === $url) {
@@ -222,8 +241,8 @@ class ManifestBuilder
 
         $modified = max(
             (int) $row['tstamp'],
-            $context['contentTimestamps'][$uid][$languageId] ?? 0,
-            $context['contentTimestamps'][$uid][-1] ?? 0
+            $content[$languageId]['tstamp'] ?? 0,
+            $content[-1]['tstamp'] ?? 0
         );
 
         return [
@@ -242,32 +261,41 @@ class ManifestBuilder
     }
 
     /**
-     * Latest change to any content element, by page and language. Deleted and hidden elements
-     * count - removing or hiding one changes the page - which is also why SYS_LASTCHANGED is no
-     * use here: it only moves when the page is rendered.
+     * Per page and language: the latest change to any content element, and how many visible
+     * elements are not plugins.
+     * Deleted and hidden elements count towards the change - removing or hiding one changes the
+     * page - which is also why SYS_LASTCHANGED is no use here: it only moves on render.
      *
-     * @return array<int, array<int, int>>
+     * @return array<int, array<int, array{tstamp: int, text: int}>>
      */
-    private function findContentTimestamps(): array
+    private function findContentStatistics(): array
     {
         $queryBuilder = $this->connectionPool->getQueryBuilderForTable('tt_content');
         $queryBuilder->getRestrictions()->removeAll();
 
         $rows = $queryBuilder
-            ->selectLiteral('pid', 'sys_language_uid', 'MAX(tstamp) AS tstamp')
+            ->selectLiteral(
+                'pid',
+                'sys_language_uid',
+                'MAX(tstamp) AS tstamp',
+                "SUM(CASE WHEN deleted = 0 AND hidden = 0 AND CType <> 'list' THEN 1 ELSE 0 END) AS text_elements"
+            )
             ->from('tt_content')
             ->groupBy('pid', 'sys_language_uid')
             ->execute()
             ->fetchAllAssociative()
         ;
 
-        $timestamps = [];
+        $statistics = [];
 
         foreach ($rows as $row) {
-            $timestamps[(int) $row['pid']][(int) $row['sys_language_uid']] = (int) $row['tstamp'];
+            $statistics[(int) $row['pid']][(int) $row['sys_language_uid']] = [
+                'tstamp' => (int) $row['tstamp'],
+                'text' => (int) $row['text_elements'],
+            ];
         }
 
-        return $timestamps;
+        return $statistics;
     }
 
     /**

public/typo3conf/ext/ep_products/Classes/ContentFeed/PageTree.php

@@ -18,7 +18,7 @@ class PageTree
     private const FIELDS = [
         'uid', 'pid', 'sorting', 'doktype', 'hidden', 'starttime', 'endtime', 'fe_group',
         'extendToSubpages', 'no_index', 'title', 'nav_title', 'description', 'shortcut',
-        'shortcut_mode', 'tstamp', 'sys_language_uid', 'l10n_parent', 'l18n_cfg',
+        'shortcut_mode', 'content_from_pid', 'tstamp', 'sys_language_uid', 'l10n_parent', 'l18n_cfg',
     ];
 
     /**

public/typo3conf/ext/ep_products/Classes/Middleware/ContentPageTypeGuardMiddleware.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace EP\EpProducts\Middleware;
+
+use EP\EpProducts\ContentFeed\ManifestBuilder;
+use EP\EpProducts\Http\ApiKeyAuthenticator;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Psr\Http\Server\MiddlewareInterface;
+use Psr\Http\Server\RequestHandlerInterface;
+use TYPO3\CMS\Core\Routing\PageArguments;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\Controller\ErrorController;
+
+/**
+ * Keeps the chatbot content-only page type (an unstyled copy of every page) off the public web:
+ * only requests carrying the CMS API key get it.
+ *
+ * Runs once the page type is known and before TSFE, so a page-cache hit cannot bypass it.
+ * staticfilecache never stores the type (`tx_staticfilecache.disableCache`); if it did, .htaccess
+ * would serve the file without this check ever running.
+ */
+class ContentPageTypeGuardMiddleware implements MiddlewareInterface
+{
+    /**
+     * @var ApiKeyAuthenticator
+     */
+    private $authenticator;
+
+    public function __construct(ApiKeyAuthenticator $authenticator)
+    {
+        $this->authenticator = $authenticator;
+    }
+
+    public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
+    {
+        $routing = $request->getAttribute('routing');
+
+        if (!$routing instanceof PageArguments
+            || (string) ManifestBuilder::CONTENT_PAGE_TYPE !== $routing->getPageType()
+        ) {
+            return $handler->handle($request);
+        }
+
+        if (!$this->authenticator->isAuthorized($request)) {
+            // 404, not 403: an unauthenticated caller should not learn the type exists.
+            return GeneralUtility::makeInstance(ErrorController::class)
+                ->pageNotFoundAction($request, 'The requested page does not exist');
+        }
+
+        return $handler->handle($request);
+    }
+}

public/typo3conf/ext/ep_products/Configuration/RequestMiddlewares.php

@@ -20,6 +20,15 @@ return [
                 'typo3/cms-frontend/site',
             ],
         ],
+        'ep/content-page-type-guard' => [
+            'target' => \EP\EpProducts\Middleware\ContentPageTypeGuardMiddleware::class,
+            'after' => [
+                'typo3/cms-frontend/page-resolver',
+            ],
+            'before' => [
+                'typo3/cms-frontend/page-argument-validator',
+            ],
+        ],
         'ep/staging-environment' => [
             'target' => \EP\EpProducts\Middleware\StagingEnvironmentMiddleware::class,
             'after' => [

public/typo3conf/ext/ep_theme/Configuration/TypoScript/Page/ContentOnly.typoscript

@@ -0,0 +1,62 @@
+# Chatbot content feed: the main content of a page, without header, navigation, footer or
+# assets. Fetched by MyE&P for pages the content manifest reports as changed; protected by
+# EP\EpProducts\Middleware\ContentPageTypeGuardMiddleware. See
+# .ddev/plans/CHATBOT-MCP-01-TYPO3-CONTENT-FEED.md §6.
+
+# Plugins (the booking widget on product detail pages), forms, sub-page menus and teaser cards
+# are left out: MyE&P has trip data first-hand, and the pages and records the cards link to are
+# in the feed in their own right. The filter applies to top-level elements only; a plugin nested
+# in a grid or container element is still rendered by its parent.
+lib.contentOnly < lib.dynamicContent
+lib.contentOnly.20.select.where = colPos={register:colPos} AND CType NOT IN ({$plugin.tx_eptheme.settings.contentOnlyExcludedCTypes})
+
+tx_eptheme_content_only = PAGE
+tx_eptheme_content_only {
+    typeNum = 1730
+    config {
+        disableAllHeaderCode = 1
+        disablePrefixComment = 1
+        additionalHeaders.10.header = Content-Type: text/html; charset=utf-8
+        additionalHeaders.20.header = X-Robots-Tag: noindex, nofollow, noarchive
+        admPanel = 0
+        debug = 0
+        # Never written to disk: the type is key-protected, and a static file would be served
+        # by .htaccess without that check ever running.
+        tx_staticfilecache.disableCache = 1
+        # Same reason for shared caches between us and the consumer: the TYPO3 page cache is
+        # fine (the guard runs before it), a proxy storing a keyed response is not.
+        sendCacheHeaders = 0
+        additionalHeaders.30.header = Cache-Control: private, no-store
+    }
+
+    10 = COA
+    10 {
+        wrap = <article>|</article>
+
+        10 = TEXT
+        10.field = title
+        10.htmlSpecialChars = 1
+        10.wrap = <h1>|</h1>
+
+        # Page templates without a sidebar render colPos 0; the sidebar layouts render 3 (header),
+        # 4 (content) and 2 (sidebar). colPos 1 holds teaser groups and is left out. The page
+        # templates pass colPos in as data; here there is no template, so each copy sets it.
+        # These must be copies (<), not references (=<): removing colPos.cObject on a reference
+        # leaves the referenced cObject in place, and every column silently renders colPos 0.
+        20 < lib.contentOnly
+        20.5.colPos.cObject >
+        20.5.colPos = 3
+
+        30 < lib.contentOnly
+        30.5.colPos.cObject >
+        30.5.colPos = 0
+
+        40 < lib.contentOnly
+        40.5.colPos.cObject >
+        40.5.colPos = 4
+
+        50 < lib.contentOnly
+        50.5.colPos.cObject >
+        50.5.colPos = 2
+    }
+}

public/typo3conf/ext/ep_theme/Configuration/TypoScript/constants.typoscript

@@ -153,6 +153,8 @@ plugin.tx_eptheme {
 		xmlSitemapRootPageUid =
 		# cat=eptheme/600/110; type=string; label=XML sitemap page UIDs to exclude
 		xmlSitemapExcludeUids =
+		# cat=eptheme/600/120; type=string; label=CTypes left out of the chatbot content-only page type (quoted, comma-separated)
+		contentOnlyExcludedCTypes = 'list','form_formframework','menu_subpages','teaser_product','teaser_hotel','teaser_concept','teaser_country','teaser_region','ep_container_teasergroup'
 
 		# cat=eptheme/300/100; type=string; label=Cookiefirst Script
 		cookieFirstScriptUrl = https://consent.cookiefirst.com/sites/ep-reisen.de-fabb1554-8932-4075-85dc-97fa68892918/consent.js