From 1b4384e3f3e6bb9dea3b526fefcaf9596d868295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Fromme?= Date: Thu, 8 Oct 2026 10:34:58 +0200 Subject: [PATCH] feat: add MCP server for the website chatbot over a mirrored TYPO3 content feed --- .env | 5 +- composer.json | 3 + composer.lock | 658 +++++++++++++++++- config/bundles.php | 1 + config/packages/framework.yaml | 7 + config/packages/http_discovery.yaml | 10 + config/packages/mcp.yaml | 16 + config/packages/monolog.yaml | 17 +- config/packages/security.yaml | 12 + config/routes/mcp.yaml | 3 + config/services.yaml | 9 + docs/chatbot-mcp.md | 112 +++ migrations/Version20261008081401.php | 32 + src/Chatbot/Mcp/Sites.php | 16 + src/Chatbot/Mcp/Tool/GetDocumentTool.php | 56 ++ src/Chatbot/Mcp/Tool/GetTripTool.php | 44 ++ src/Chatbot/Mcp/Tool/SearchContentTool.php | 47 ++ src/Chatbot/Mcp/Tool/SearchTripsTool.php | 97 +++ src/Chatbot/Mcp/ToolCallRunner.php | 71 ++ src/Chatbot/Search/ContentSearch.php | 85 +++ src/Chatbot/Search/ContentSearchQuery.php | 66 ++ src/Chatbot/Search/Excerpt.php | 53 ++ src/Chatbot/TripCatalog.php | 653 +++++++++++++++++ src/Command/ContentCheckFreshnessCommand.php | 63 ++ src/Command/ContentSyncCommand.php | 83 +++ src/ContentFeed/ContentFeedClient.php | 312 +++++++++ src/ContentFeed/ContentSyncer.php | 367 ++++++++++ src/ContentFeed/DocumentBodyBuilder.php | 288 ++++++++ .../Exception/ContentFeedStoppedException.php | 15 + .../ContentFeedUnavailableException.php | 16 + .../ContentRequestFailedException.php | 13 + .../Exception/PagePhaseEndedException.php | 13 + src/ContentFeed/HtmlToMarkdownConverter.php | 105 +++ src/Entity/ContentDocument.php | 325 +++++++++ src/Entity/ContentSyncState.php | 97 +++ src/Enum/ContentFetchState.php | 23 + src/Model/Chatbot/TripCriteria.php | 26 + src/Model/ContentFeed/ContentManifest.php | 23 + src/Model/ContentFeed/ContentSyncReport.php | 42 ++ src/Model/ContentFeed/ManifestItem.php | 90 +++ src/Model/ContentFeed/PageFetchResult.php | 41 ++ src/Repository/ContentDocumentRepository.php | 166 +++++ src/Repository/ContentSyncStateRepository.php | 28 + src/Security/McpAccessTokenHandler.php | 37 + symfony.lock | 15 + .../Chatbot/Search/ContentSearchQueryTest.php | 47 ++ tests/Chatbot/TripCatalogTest.php | 171 +++++ tests/ContentFeed/ContentFeedClientTest.php | 257 +++++++ tests/ContentFeed/ContentSyncerTest.php | 227 ++++++ tests/ContentFeed/DocumentBodyBuilderTest.php | 74 ++ .../HtmlToMarkdownConverterTest.php | 53 ++ tests/Security/McpAccessTokenHandlerTest.php | 31 + 52 files changed, 5117 insertions(+), 4 deletions(-) create mode 100644 config/packages/http_discovery.yaml create mode 100644 config/packages/mcp.yaml create mode 100644 config/routes/mcp.yaml create mode 100644 docs/chatbot-mcp.md create mode 100644 migrations/Version20261008081401.php create mode 100644 src/Chatbot/Mcp/Sites.php create mode 100644 src/Chatbot/Mcp/Tool/GetDocumentTool.php create mode 100644 src/Chatbot/Mcp/Tool/GetTripTool.php create mode 100644 src/Chatbot/Mcp/Tool/SearchContentTool.php create mode 100644 src/Chatbot/Mcp/Tool/SearchTripsTool.php create mode 100644 src/Chatbot/Mcp/ToolCallRunner.php create mode 100644 src/Chatbot/Search/ContentSearch.php create mode 100644 src/Chatbot/Search/ContentSearchQuery.php create mode 100644 src/Chatbot/Search/Excerpt.php create mode 100644 src/Chatbot/TripCatalog.php create mode 100644 src/Command/ContentCheckFreshnessCommand.php create mode 100644 src/Command/ContentSyncCommand.php create mode 100644 src/ContentFeed/ContentFeedClient.php create mode 100644 src/ContentFeed/ContentSyncer.php create mode 100644 src/ContentFeed/DocumentBodyBuilder.php create mode 100644 src/ContentFeed/Exception/ContentFeedStoppedException.php create mode 100644 src/ContentFeed/Exception/ContentFeedUnavailableException.php create mode 100644 src/ContentFeed/Exception/ContentRequestFailedException.php create mode 100644 src/ContentFeed/Exception/PagePhaseEndedException.php create mode 100644 src/ContentFeed/HtmlToMarkdownConverter.php create mode 100644 src/Entity/ContentDocument.php create mode 100644 src/Entity/ContentSyncState.php create mode 100644 src/Enum/ContentFetchState.php create mode 100644 src/Model/Chatbot/TripCriteria.php create mode 100644 src/Model/ContentFeed/ContentManifest.php create mode 100644 src/Model/ContentFeed/ContentSyncReport.php create mode 100644 src/Model/ContentFeed/ManifestItem.php create mode 100644 src/Model/ContentFeed/PageFetchResult.php create mode 100644 src/Repository/ContentDocumentRepository.php create mode 100644 src/Repository/ContentSyncStateRepository.php create mode 100644 src/Security/McpAccessTokenHandler.php create mode 100644 tests/Chatbot/Search/ContentSearchQueryTest.php create mode 100644 tests/Chatbot/TripCatalogTest.php create mode 100644 tests/ContentFeed/ContentFeedClientTest.php create mode 100644 tests/ContentFeed/ContentSyncerTest.php create mode 100644 tests/ContentFeed/DocumentBodyBuilderTest.php create mode 100644 tests/ContentFeed/HtmlToMarkdownConverterTest.php create mode 100644 tests/Security/McpAccessTokenHandlerTest.php diff --git a/.env b/.env index a9c336f0..87985b5f 100644 --- a/.env +++ b/.env @@ -82,8 +82,11 @@ APP_LEGAL_NOTICE_URL="https://www.ep-reisen.de/unternehmen/impressum-skireisever APP_TERMS_AND_CONDITIONS_URL="https://www.ep-reisen.de/skireisen/infos/allgemeine-infos/agb/" APP_PRIVACY_POLICY_URL="https://www.ep-reisen.de/unternehmen/impressum-skireiseveranstalter/datenschutz-winterreisen/" APP_CMS_API_BASE_URL="https://www.ep-reisen.de/" -# TYPO3 API key (X-Api-Key), sent by the typo3.client scoped client; the real value per stage goes in .env.local +# TYPO3 API key (X-Api-Key), sent by the typo3.client scoped client and by app:content:sync; +# the real value per stage goes in .env.local APP_CMS_API_KEY= +# Static bearer token n8n sends to the chatbot MCP endpoint (/mcp); empty refuses every request +APP_MCP_TOKEN= APP_WEBSITE_BASE_URL="https://www.ep-reisen.de/" APP_BASE_URL="https://my.ep-reisen.de" APP_CUSTOMER_SERVICE_EMAIL="kundenservice@ep-reisen.de" diff --git a/composer.json b/composer.json index ba8bce16..97deb477 100644 --- a/composer.json +++ b/composer.json @@ -21,6 +21,7 @@ "knplabs/knp-paginator-bundle": "^6.10", "league/flysystem-bundle": "^3.4", "league/flysystem-sftp-v3": "^3.29", + "league/html-to-markdown": "^5.1", "league/oauth2-server-bundle": "^1.0", "maennchen/zipstream-php": "^3.2", "nesbot/carbon": "^3.8", @@ -43,12 +44,14 @@ "symfony/intl": "7.4.*", "symfony/lock": "7.4.*", "symfony/mailer": "7.4.*", + "symfony/mcp-bundle": "^0.14.1", "symfony/mime": "7.4.*", "symfony/monolog-bundle": "^3.0", "symfony/notifier": "7.4.*", "symfony/process": "7.4.*", "symfony/property-access": "7.4.*", "symfony/property-info": "7.4.*", + "symfony/rate-limiter": "7.4.*", "symfony/runtime": "7.4.*", "symfony/security-bundle": "7.4.*", "symfony/serializer": "7.4.*", diff --git a/composer.lock b/composer.lock index f47144fa..fc00ec4e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2c2a8c19ca2e117f521584a4e3e6212c", + "content-hash": "c21fb67f385dd7290a440aec1a70cb9f", "packages": [ { "name": "carbonphp/carbon-doctrine-types", @@ -2521,6 +2521,95 @@ }, "time": "2026-09-02T07:49:56+00:00" }, + { + "name": "league/html-to-markdown", + "version": "5.1.3", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/html-to-markdown.git", + "reference": "5411b9650dd468be52267a6f4a2a26c31d2581be" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/5411b9650dd468be52267a6f4a2a26c31d2581be", + "reference": "5411b9650dd468be52267a6f4a2a26c31d2581be", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-xml": "*", + "php": "^7.2.5 || ^8.0" + }, + "require-dev": { + "mikehaertl/php-shellcommand": "^1.1.0", + "phpstan/phpstan": "^1.8.8 || ^2.0", + "phpunit/phpunit": "^8.5 || ^9.2 || ^10.5 || ^11.0 || ^12.0 || ^13.0", + "scrutinizer/ocular": "^1.6", + "unleashedtech/php-coding-standard": "^2.7 || ^3.0", + "vimeo/psalm": "^4.22 || ^5.0 || ^6.0" + }, + "bin": [ + "bin/html-to-markdown" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.2-dev" + } + }, + "autoload": { + "psr-4": { + "League\\HTMLToMarkdown\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Colin O'Dell", + "email": "colinodell@gmail.com", + "homepage": "https://www.colinodell.com", + "role": "Lead Developer" + }, + { + "name": "Nick Cernis", + "email": "nick@cern.is", + "homepage": "http://modernnerd.net", + "role": "Original Author" + } + ], + "description": "An HTML-to-markdown conversion helper for PHP", + "homepage": "https://github.com/thephpleague/html-to-markdown", + "keywords": [ + "html", + "markdown" + ], + "support": { + "issues": "https://github.com/thephpleague/html-to-markdown/issues", + "source": "https://github.com/thephpleague/html-to-markdown/tree/5.1.3" + }, + "funding": [ + { + "url": "https://www.colinodell.com/sponsor", + "type": "custom" + }, + { + "url": "https://www.paypal.me/colinpodell/10.00", + "type": "custom" + }, + { + "url": "https://github.com/colinodell", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/league/html-to-markdown", + "type": "tidelift" + } + ], + "time": "2026-10-06T01:08:41+00:00" + }, { "name": "league/mime-type-detection", "version": "1.17.0", @@ -3192,6 +3281,91 @@ }, "time": "2026-08-18T06:18:41+00:00" }, + { + "name": "mcp/sdk", + "version": "v0.8.1", + "source": { + "type": "git", + "url": "https://github.com/modelcontextprotocol/php-sdk.git", + "reference": "c5dbfb64e5a2a30872ec3927b4cf13a2eb05e10e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/modelcontextprotocol/php-sdk/zipball/c5dbfb64e5a2a30872ec3927b4cf13a2eb05e10e", + "reference": "c5dbfb64e5a2a30872ec3927b4cf13a2eb05e10e", + "shasum": "" + }, + "require": { + "ext-fileinfo": "*", + "opis/json-schema": "^2.4", + "php": "^8.1", + "php-http/discovery": "^1.20", + "phpdocumentor/reflection-docblock": "^5.6 || ^6.0", + "psr/clock": "^1.0", + "psr/container": "^1.0 || ^2.0", + "psr/event-dispatcher": "^1.0", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.1", + "psr/http-message": "^1.1 || ^2.0", + "psr/http-server-handler": "^1.0", + "psr/http-server-middleware": "^1.0", + "psr/log": "^1.0 || ^2.0 || ^3.0", + "symfony/deprecation-contracts": "^2.5 || ^3.0", + "symfony/uid": "^5.4 || ^6.4 || ^7.3 || ^8.0" + }, + "require-dev": { + "composer/semver": "^3.0", + "ext-openssl": "*", + "firebase/php-jwt": "^6.10 || ^7.0", + "laminas/laminas-httphandlerrunner": "^2.12", + "nyholm/psr7": "^1.8", + "nyholm/psr7-server": "^1.1", + "phar-io/composer-distributor": "^1.0.2", + "php-cs-fixer/shim": "^3.91", + "phpdocumentor/shim": "^3", + "phpstan/phpstan": "^2.1", + "phpunit/phpunit": "^10.5", + "psr/simple-cache": "^2.0 || ^3.0", + "symfony/cache": "^5.4 || ^6.4 || ^7.3 || ^8.0", + "symfony/console": "^5.4 || ^6.4 || ^7.3 || ^8.0", + "symfony/finder": "^5.4 || ^6.4 || ^7.3 || ^8.0", + "symfony/http-client": "^5.4 || ^6.4 || ^7.3 || ^8.0", + "symfony/process": "^5.4 || ^6.4 || ^7.3 || ^8.0" + }, + "suggest": { + "symfony/finder": "Required for file-based discovery." + }, + "type": "library", + "autoload": { + "psr-4": { + "Mcp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Christopher Hertel", + "email": "mail@christopher-hertel.de" + }, + { + "name": "Kyrian Obikwelu", + "email": "koshnawaza@gmail.com" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com" + } + ], + "description": "Model Context Protocol SDK for Client and Server applications in PHP", + "support": { + "issues": "https://github.com/modelcontextprotocol/php-sdk/issues", + "source": "https://github.com/modelcontextprotocol/php-sdk/tree/v0.8.1" + }, + "time": "2026-08-29T23:10:56+00:00" + }, { "name": "monolog/monolog", "version": "3.12.1", @@ -3480,6 +3654,196 @@ ], "time": "2024-09-09T07:06:30+00:00" }, + { + "name": "opis/json-schema", + "version": "2.6.0", + "source": { + "type": "git", + "url": "https://github.com/opis/json-schema.git", + "reference": "8458763e0dd0b6baa310e04f1829fc73da4e8c8a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/json-schema/zipball/8458763e0dd0b6baa310e04f1829fc73da4e8c8a", + "reference": "8458763e0dd0b6baa310e04f1829fc73da4e8c8a", + "shasum": "" + }, + "require": { + "ext-json": "*", + "opis/string": "^2.1", + "opis/uri": "^1.0", + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "ext-bcmath": "*", + "ext-intl": "*", + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-4": { + "Opis\\JsonSchema\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + }, + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + } + ], + "description": "Json Schema Validator for PHP", + "homepage": "https://opis.io/json-schema", + "keywords": [ + "json", + "json-schema", + "schema", + "validation", + "validator" + ], + "support": { + "issues": "https://github.com/opis/json-schema/issues", + "source": "https://github.com/opis/json-schema/tree/2.6.0" + }, + "time": "2025-10-17T12:46:48+00:00" + }, + { + "name": "opis/string", + "version": "2.1.0", + "source": { + "type": "git", + "url": "https://github.com/opis/string.git", + "reference": "3e4d2aaff518ac518530b89bb26ed40f4503635e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/string/zipball/3e4d2aaff518ac518530b89bb26ed40f4503635e", + "reference": "3e4d2aaff518ac518530b89bb26ed40f4503635e", + "shasum": "" + }, + "require": { + "ext-iconv": "*", + "ext-json": "*", + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.x-dev" + } + }, + "autoload": { + "psr-4": { + "Opis\\String\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + }, + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + } + ], + "description": "Multibyte strings as objects", + "homepage": "https://opis.io/string", + "keywords": [ + "multi-byte", + "opis", + "string", + "string manipulation", + "utf-8" + ], + "support": { + "issues": "https://github.com/opis/string/issues", + "source": "https://github.com/opis/string/tree/2.1.0" + }, + "time": "2025-10-17T12:38:41+00:00" + }, + { + "name": "opis/uri", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/opis/uri.git", + "reference": "0f3ca49ab1a5e4a6681c286e0b2cc081b93a7d5a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/uri/zipball/0f3ca49ab1a5e4a6681c286e0b2cc081b93a7d5a", + "reference": "0f3ca49ab1a5e4a6681c286e0b2cc081b93a7d5a", + "shasum": "" + }, + "require": { + "opis/string": "^2.0", + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Opis\\Uri\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + }, + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + } + ], + "description": "Build, parse and validate URIs and URI-templates", + "homepage": "https://opis.io", + "keywords": [ + "URI Template", + "parse url", + "punycode", + "uri", + "uri components", + "url", + "validate uri" + ], + "support": { + "issues": "https://github.com/opis/uri/issues", + "source": "https://github.com/opis/uri/tree/1.1.0" + }, + "time": "2021-05-22T15:57:08+00:00" + }, { "name": "paragonie/constant_time_encoding", "version": "v3.1.3", @@ -3599,6 +3963,85 @@ }, "time": "2020-10-15T08:29:30+00:00" }, + { + "name": "php-http/discovery", + "version": "1.20.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/discovery.git", + "reference": "82fe4c73ef3363caed49ff8dd1539ba06044910d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/discovery/zipball/82fe4c73ef3363caed49ff8dd1539ba06044910d", + "reference": "82fe4c73ef3363caed49ff8dd1539ba06044910d", + "shasum": "" + }, + "require": { + "composer-plugin-api": "^1.0|^2.0", + "php": "^7.1 || ^8.0" + }, + "conflict": { + "nyholm/psr7": "<1.0", + "zendframework/zend-diactoros": "*" + }, + "provide": { + "php-http/async-client-implementation": "*", + "php-http/client-implementation": "*", + "psr/http-client-implementation": "*", + "psr/http-factory-implementation": "*", + "psr/http-message-implementation": "*" + }, + "require-dev": { + "composer/composer": "^1.0.2|^2.0", + "graham-campbell/phpspec-skip-example-extension": "^5.0", + "php-http/httplug": "^1.0 || ^2.0", + "php-http/message-factory": "^1.0", + "phpspec/phpspec": "^5.1 || ^6.1 || ^7.3", + "sebastian/comparator": "^3.0.5 || ^4.0.8", + "symfony/phpunit-bridge": "^6.4.4 || ^7.0.1" + }, + "type": "composer-plugin", + "extra": { + "class": "Http\\Discovery\\Composer\\Plugin", + "plugin-optional": true + }, + "autoload": { + "psr-4": { + "Http\\Discovery\\": "src/" + }, + "exclude-from-classmap": [ + "src/Composer/Plugin.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Finds and installs PSR-7, PSR-17, PSR-18 and HTTPlug implementations", + "homepage": "http://php-http.org", + "keywords": [ + "adapter", + "client", + "discovery", + "factory", + "http", + "message", + "psr17", + "psr7" + ], + "support": { + "issues": "https://github.com/php-http/discovery/issues", + "source": "https://github.com/php-http/discovery/tree/1.20.0" + }, + "time": "2024-10-02T11:20:13+00:00" + }, { "name": "phpdocumentor/reflection-common", "version": "2.2.1", @@ -4247,6 +4690,58 @@ }, "time": "2019-01-08T18:20:26+00:00" }, + { + "name": "psr/http-client", + "version": "1.0.3", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-client.git", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-client/zipball/bb5906edc1c324c9a05aa0873d40117941e5fa90", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP clients", + "homepage": "https://github.com/php-fig/http-client", + "keywords": [ + "http", + "http-client", + "psr", + "psr-18" + ], + "support": { + "source": "https://github.com/php-fig/http-client" + }, + "time": "2023-09-23T14:17:50+00:00" + }, { "name": "psr/http-factory", "version": "1.1.0", @@ -7296,6 +7791,93 @@ ], "time": "2026-09-15T06:01:15+00:00" }, + { + "name": "symfony/mcp-bundle", + "version": "v0.14.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/mcp-bundle.git", + "reference": "f9f5be5e16a7b4749ca98d8099042902fe917a74" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/mcp-bundle/zipball/f9f5be5e16a7b4749ca98d8099042902fe917a74", + "reference": "f9f5be5e16a7b4749ca98d8099042902fe917a74", + "shasum": "" + }, + "require": { + "mcp/sdk": "^0.8.1", + "php-http/discovery": "^1.20", + "symfony/config": "^7.3|^8.0", + "symfony/console": "^7.3|^8.0", + "symfony/dependency-injection": "^7.3|^8.0", + "symfony/finder": "^7.3|^8.0", + "symfony/framework-bundle": "^7.3|^8.0", + "symfony/http-foundation": "^7.3|^8.0", + "symfony/http-kernel": "^7.3|^8.0", + "symfony/psr-http-message-bridge": "^7.3|^8.0", + "symfony/routing": "^7.3|^8.0", + "symfony/service-contracts": "^2.5|^3" + }, + "require-dev": { + "nyholm/psr7": "^1.8", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpstan/phpstan-strict-rules": "^2.0", + "phpunit/phpunit": "^11.5.53", + "symfony/monolog-bundle": "^3.10 || ^4.0", + "symfony/twig-bundle": "^7.3|^8.0" + }, + "type": "symfony-bundle", + "extra": { + "thanks": { + "url": "https://github.com/symfony/ai", + "name": "symfony/ai" + } + }, + "autoload": { + "psr-4": { + "Symfony\\AI\\McpBundle\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christopher Hertel", + "email": "mail@christopher-hertel.de" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony integration bundle for Model Context Protocol (via official mcp/sdk)", + "support": { + "source": "https://github.com/symfony/mcp-bundle/tree/v0.14.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-09-26T01:53:23+00:00" + }, { "name": "symfony/messenger", "version": "v7.4.20", @@ -9028,6 +9610,80 @@ ], "time": "2026-03-24T13:12:05+00:00" }, + { + "name": "symfony/rate-limiter", + "version": "v7.4.18", + "source": { + "type": "git", + "url": "https://github.com/symfony/rate-limiter.git", + "reference": "6703d040ab83401b27f3c7b29ff4c8c8106fedba" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/rate-limiter/zipball/6703d040ab83401b27f3c7b29ff4c8c8106fedba", + "reference": "6703d040ab83401b27f3c7b29ff4c8c8106fedba", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "symfony/options-resolver": "^7.3|^8.0" + }, + "require-dev": { + "psr/cache": "^1.0|^2.0|^3.0", + "symfony/lock": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\RateLimiter\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Wouter de Jong", + "email": "wouter@wouterj.nl" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides a Token Bucket implementation to rate limit input and output in your application", + "homepage": "https://symfony.com", + "keywords": [ + "limiter", + "rate-limiter" + ], + "support": { + "source": "https://github.com/symfony/rate-limiter/tree/v7.4.18" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-08-07T15:34:54+00:00" + }, { "name": "symfony/routing", "version": "v7.4.20", diff --git a/config/bundles.php b/config/bundles.php index 7bc0cb7f..544070a1 100644 --- a/config/bundles.php +++ b/config/bundles.php @@ -20,4 +20,5 @@ return [ Knp\Bundle\MenuBundle\KnpMenuBundle::class => ['all' => true], Knp\Bundle\PaginatorBundle\KnpPaginatorBundle::class => ['all' => true], Symfony\UX\TwigComponent\TwigComponentBundle::class => ['all' => true], + Symfony\AI\McpBundle\McpBundle::class => ['all' => true], ]; diff --git a/config/packages/framework.yaml b/config/packages/framework.yaml index 57f7db6f..5130987c 100644 --- a/config/packages/framework.yaml +++ b/config/packages/framework.yaml @@ -16,6 +16,13 @@ framework: php_errors: log: true + rate_limiter: + # Per MCP token. A chatbot turn makes a few tool calls; this catches runaway loops. + chatbot_mcp: + policy: sliding_window + limit: 60 + interval: '1 minute' + http_client: scoped_clients: typo3.client: diff --git a/config/packages/http_discovery.yaml b/config/packages/http_discovery.yaml new file mode 100644 index 00000000..2a789e73 --- /dev/null +++ b/config/packages/http_discovery.yaml @@ -0,0 +1,10 @@ +services: + Psr\Http\Message\RequestFactoryInterface: '@http_discovery.psr17_factory' + Psr\Http\Message\ResponseFactoryInterface: '@http_discovery.psr17_factory' + Psr\Http\Message\ServerRequestFactoryInterface: '@http_discovery.psr17_factory' + Psr\Http\Message\StreamFactoryInterface: '@http_discovery.psr17_factory' + Psr\Http\Message\UploadedFileFactoryInterface: '@http_discovery.psr17_factory' + Psr\Http\Message\UriFactoryInterface: '@http_discovery.psr17_factory' + + http_discovery.psr17_factory: + class: Http\Discovery\Psr17Factory diff --git a/config/packages/mcp.yaml b/config/packages/mcp.yaml new file mode 100644 index 00000000..41951882 --- /dev/null +++ b/config/packages/mcp.yaml @@ -0,0 +1,16 @@ +# The website chatbot's MCP server, called by the n8n AI Agent through its MCP Client Tool +# node. Read-only tools over the mirrored TYPO3 content and our own trip data; see +# docs/chatbot-mcp.md. Authentication is the "mcp" firewall in security.yaml. +mcp: + servers: + chatbot: + name: 'MyE&P chatbot' + version: '1.0.0' + instructions: 'Answers questions about the trips and website content of E&P Reisen and its brands. Every result carries a url to cite. Prices and availability are a snapshot as of dataAsOf; the booking page is binding.' + http: + path: /mcp + # DNS rebinding protection guards unauthenticated local servers. This one is + # public, behind a bearer token, and never called from a browser. + allowed_hosts: false + registry: + tools: ['App\Chatbot\Mcp\Tool\'] diff --git a/config/packages/monolog.yaml b/config/packages/monolog.yaml index 71ea5d9e..431e113f 100644 --- a/config/packages/monolog.yaml +++ b/config/packages/monolog.yaml @@ -5,6 +5,7 @@ monolog: - core - auth - admin + - chatbot when@dev: monolog: @@ -33,11 +34,16 @@ when@dev: path: "%kernel.logs_dir%/%kernel.environment%.admin.log" level: debug channels: ["admin"] + chatbot: + type: stream + path: "%kernel.logs_dir%/%kernel.environment%.chatbot.log" + level: debug + channels: ["chatbot"] main: type: stream path: "%kernel.logs_dir%/%kernel.environment%.log" level: debug - channels: ["!event", "!bpn", "!core", "!auth", "!admin"] + channels: ["!event", "!bpn", "!core", "!auth", "!admin", "!chatbot"] # uncomment to get logging in your browser # you may have to allow bigger header sizes in your Web server configuration #firephp: @@ -96,13 +102,20 @@ when@prod: &prod_config path: "%kernel.logs_dir%/%kernel.environment%.admin.log" level: info channels: ["admin"] + # MCP tool calls, file only: the log for judging search quality later. + chatbot: + type: rotating_file + max_files: 30 + path: "%kernel.logs_dir%/%kernel.environment%.chatbot.log" + level: info + channels: ["chatbot"] main: type: fingers_crossed action_level: error handler: nested excluded_http_codes: [404, 405] buffer_size: 50 # How many messages should be saved? Prevent memory leaks - channels: ["!bpn", "!core", "!auth", "!admin"] + channels: ["!bpn", "!core", "!auth", "!admin", "!chatbot"] nested: type: rotating_file path: "%kernel.logs_dir%/%kernel.environment%.framework.log" diff --git a/config/packages/security.yaml b/config/packages/security.yaml index e96018fd..1a5e968c 100644 --- a/config/packages/security.yaml +++ b/config/packages/security.yaml @@ -15,6 +15,11 @@ security: mailjet: password: '%env(MAILJET_WEBHOOK_BASIC_PASSWORD)%' roles: [ ROLE_MAILJET_WEBHOOK ] + mcp_provider: + memory: + users: + n8n: + roles: [ ROLE_CHATBOT_MCP ] role_hierarchy: # ROLE_CUSTOMER_EXPERT gates the booking-draft and log surfaces, which used to be @@ -45,6 +50,12 @@ security: provider: mailjet_webhook_provider http_basic: realm: 'Mailjet Webhook' + mcp: + pattern: ^/mcp$ + stateless: true + provider: mcp_provider + access_token: + token_handler: App\Security\McpAccessTokenHandler main: lazy: true provider: app_user_provider @@ -58,6 +69,7 @@ security: # Note: Only the *first* access control that matches will be used access_control: - { path: ^/webhooks/mailjet/newsletter$, roles: ROLE_MAILJET_WEBHOOK, requires_channel: https } + - { path: ^/mcp$, roles: ROLE_CHATBOT_MCP, requires_channel: https } - { path: ^/authorize, roles: IS_AUTHENTICATED_REMEMBERED, requires_channel: https } - { path: ^/, roles: PUBLIC_ACCESS, requires_channel: https } diff --git a/config/routes/mcp.yaml b/config/routes/mcp.yaml new file mode 100644 index 00000000..2a14e010 --- /dev/null +++ b/config/routes/mcp.yaml @@ -0,0 +1,3 @@ +mcp: + resource: . + type: mcp diff --git a/config/services.yaml b/config/services.yaml index 65168946..4a0a3877 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -333,6 +333,15 @@ services: - '@App\Dashboard\Widget\PendingRoleApprovalsWidgetProvider' - '@App\Dashboard\Widget\RecentLogEntriesWidgetProvider' + App\ContentFeed\ContentFeedClient: + arguments: + $baseUrl: '%env(APP_CMS_API_BASE_URL)%' + $apiKey: '%env(APP_CMS_API_KEY)%' + + App\Security\McpAccessTokenHandler: + arguments: + $token: '%env(APP_MCP_TOKEN)%' + App\Service\CmsDataProvider: arguments: $httpClient: '@typo3.client' diff --git a/docs/chatbot-mcp.md b/docs/chatbot-mcp.md new file mode 100644 index 00000000..d1738e6e --- /dev/null +++ b/docs/chatbot-mcp.md @@ -0,0 +1,112 @@ +# Chatbot MCP server + +The website chatbot runs in n8n. Its AI Agent answers from a read-only MCP server in this app instead +of scraping the websites. The server combines two sources: + +- **Editorial content from TYPO3**: the content feed of all public sites (ep-reisen, ser, + ski-boarderweek, ski-wochenenden, snowzone), mirrored into `content_document` by `app:content:sync`. +- **Trip data of our own**: dates, rooms, prices, availability and pickups from the product export + (`ProductExportBuilder`, the data behind `GET /api/exports/products`). + +TYPO3 sends no dates or prices, and nothing flows back to TYPO3. + +## Endpoint + +`POST /mcp` (Streamable HTTP; `GET`/`DELETE` are part of the transport), served by `symfony/mcp-bundle` +from `config/packages/mcp.yaml`. Tools only, no resources or prompts. + +Authentication is a static bearer token, `Authorization: Bearer `, checked by +`App\Security\McpAccessTokenHandler` on the `mcp` firewall. If `APP_MCP_TOKEN` is empty, every request is +refused. It is not the OAuth2 client-credentials flow of `/api`: n8n's MCP Client Tool node sends a fixed +header and cannot renew a token every 10 minutes. + +Tool calls are rate-limited per token (`chatbot_mcp` limiter, 60 per minute). The limit is there to stop +runaway agent loops. Each call is logged on the `chatbot` channel (`var/log/.chatbot.log`, file +only) with the tool, arguments, result count and duration. Nothing identifies the person chatting. + +### n8n + +MCP Client Tool node: endpoint `https:///mcp`, transport *HTTP Streamable*, authentication *Bearer* +with the stage's `APP_MCP_TOKEN`. The agent prompt should tell the bot to cite each answer's `url`, and +that prices and availability are a snapshot (`dataAsOf`) and only the booking page is binding. + +## Tools + +| Tool | Use | Returns | +|---|---|---| +| `search_trips` | Which trips exist, when, where, how much | Up to 10 trips with their next 3 matching dates, `priceFrom`, availability, `dataAsOf` | +| `get_trip` | One trip by `code` (optionally `hotelCode`) | Product text with FAQs, hotels with facts, all upcoming dates with bookable rooms, pickups for the first 3 dates | +| `search_content` | Questions not about dates or prices | Up to 8 hits with a ~600-character excerpt | +| `get_document` | The full text of one `search_content` hit, by `url` | Markdown body, truncated at 20 000 characters | + +Every result carries the public `url`. Output is compact JSON with Markdown in the text fields. +An empty result is `{"results": []}`. Invalid input produces a tool error the model can read. + +Trip rules (`App\Chatbot\TripCatalog`): + +- TYPO3 products are joined to the export on `product.code`. Several products can share a code, and + each is listed as its own trip. Hotels are joined on any entry of the TYPO3 hotel's `codes` (BusPro + uses the matchcode aliases). A code that is one hotel's own and another's alias resolves to its owner. +- A room is bookable unless its status is `Buchungsstop`. `available` counts rooms, so the places of a + date at a hotel are `Σ available × maxPax` over its bookable rooms. `persons` filters on that sum, and + `maxPrice` on the lowest bookable room price. The tools never output room counts, only `frei` or + `auf Anfrage`. +- `search_trips` only offers products that TYPO3 lists in its own website search (`searchable`) and does + not mark as unbookable. Closed group trips and staff trainings are `searchable: false`. `get_trip` still + answers when someone names their code. +- The export's `guide` (name, phone) and all BusPro ids are never output. + +Search (`App\Chatbot\Search\ContentSearch`) is MariaDB full text in boolean mode over +`title, breadcrumb, body_md`. Each term becomes a prefix match, because InnoDB has no German stemming, +and German stopwords are dropped in PHP. Relevance is weighted by type: products and hotels, then +destinations and concepts, then pages, then blog posts. + +## Content sync + +```bash +php bin/console app:content:sync # delta: what the manifest says changed +php bin/console app:content:sync --refresh # also refetch items older than 7 days, oldest first +php bin/console app:content:sync --ignore-window # deliberate bulk run outside 01:00–05:00 +php bin/console app:content:check-freshness # fails if the last good manifest is > 60 min old +``` + +Not scheduled yet. The intended schedule is a delta sync every 15 minutes, `--refresh` on Sunday at +01:00, and the freshness check hourly, so that `email_on_failure` reports a stalled sync. + +The feed is read with `X-Api-Key: ` from `APP_CMS_API_BASE_URL`. The manifest lists +every site whichever host serves it. The key is only sent to hosts of the sites the manifest names. + +**Load rules.** These are agreed with the TYPO3 side, so do not tune them: + +- One request at a time, and at most one page fetch per second. +- A response slower than 2 s makes the next request wait 5 s. Three slow pages in a row end the page + phase for the run. +- A `503` is TYPO3's kill switch. The run stops, and `Retry-After` pauses later runs (`content_sync_state.paused_until`). + Three consecutive 5xx responses or timeouts (10 s) also stop the run. Nothing is retried in a loop. +- Outside 01:00–05:00 Europe/Berlin a run fetches at most 50 pages and leaves the rest for later. + +**Robustness.** + +- A failed, invalid or empty manifest changes nothing. +- A record that is requested but absent from the records response is deleted, and so is an item that + is missing from the manifest. +- A page answering 404 or 5xx is remembered as `failed`, and one that converts to no text as `empty`. + Each is stored with its `modified` and not refetched until that changes or a `--refresh` reaches it. +- A timeout is retried on the next run. + +**Document model.** One `content_document` row per `(type, uid, language)`. A record's structured +payload is kept in `data`, and HTML is converted to Markdown (`HtmlToMarkdownConverter`). A page whose +`detailOf` names a record is appended to that record's `body_md` and kept out of the index +(`merged_into`), so a product is never found twice. Merges are recomputed on every run. A record without +a reachable page (`site`/`url` null) is not shown in search results, unless a detail page lends it its URL. + +After a complete run that changed something, the codes the trip join cannot place are logged on `core` +(`Content feed: unmatched codes`). + +## Configuration + +| Variable | Meaning | +|---|---| +| `APP_CMS_API_BASE_URL` | TYPO3 base URL per stage (also used by `CmsDataProvider`) | +| `APP_CMS_API_KEY` | TYPO3 `X-Api-Key`: the content feed and its `contentUrl` pages, and `typo3.client` (`CmsDataProvider`) | +| `APP_MCP_TOKEN` | Bearer token n8n sends to `/mcp` | diff --git a/migrations/Version20261008081401.php b/migrations/Version20261008081401.php new file mode 100644 index 00000000..b49b2de6 --- /dev/null +++ b/migrations/Version20261008081401.php @@ -0,0 +1,32 @@ +addSql('CREATE TABLE content_document (id INT AUTO_INCREMENT NOT NULL, type VARCHAR(16) NOT NULL, uid INT NOT NULL, language VARCHAR(8) NOT NULL, site VARCHAR(32) DEFAULT NULL, url VARCHAR(768) DEFAULT NULL, title VARCHAR(255) NOT NULL, breadcrumb VARCHAR(512) DEFAULT NULL, body_md MEDIUMTEXT NOT NULL, data JSON DEFAULT NULL COMMENT \'(DC2Type:json)\', modified DATETIME NOT NULL COMMENT \'(DC2Type:datetime_immutable)\', fetched_modified DATETIME DEFAULT NULL COMMENT \'(DC2Type:datetime_immutable)\', fetch_state VARCHAR(16) NOT NULL, synced_at DATETIME DEFAULT NULL COMMENT \'(DC2Type:datetime_immutable)\', merged_into VARCHAR(32) DEFAULT NULL, indexed TINYINT(1) NOT NULL, INDEX content_document_indexed_site (indexed, site), INDEX content_document_url (url), FULLTEXT INDEX content_document_fulltext (title, breadcrumb, body_md), UNIQUE INDEX content_document_identity (type, uid, language), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB'); + $this->addSql('CREATE TABLE content_sync_state (id INT AUTO_INCREMENT NOT NULL, manifest_generated_at DATETIME DEFAULT NULL COMMENT \'(DC2Type:datetime_immutable)\', last_run_at DATETIME DEFAULT NULL COMMENT \'(DC2Type:datetime_immutable)\', last_error LONGTEXT DEFAULT NULL, paused_until DATETIME DEFAULT NULL COMMENT \'(DC2Type:datetime_immutable)\', PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8mb4 COLLATE `utf8mb4_unicode_ci` ENGINE = InnoDB'); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE content_document'); + $this->addSql('DROP TABLE content_sync_state'); + } +} diff --git a/src/Chatbot/Mcp/Sites.php b/src/Chatbot/Mcp/Sites.php new file mode 100644 index 00000000..f7041cf3 --- /dev/null +++ b/src/Chatbot/Mcp/Sites.php @@ -0,0 +1,16 @@ +runner->run('get_document', get_defined_vars(), function () use ($url): array { + $document = $this->repository->findOneIndexedByUrl(trim($url)); + + if (null === $document) { + throw new ToolCallException('No document at this url. Use a url returned by search_content.'); + } + + $body = $document->getBodyMd(); + $truncated = mb_strlen($body) > self::BODY_LIMIT; + + return [ + 'type' => ContentSearch::publicType($document->getType()), + 'title' => $document->getTitle(), + 'url' => $document->getUrl(), + 'site' => $document->getSite(), + 'breadcrumb' => $document->getBreadcrumb(), + 'body' => $truncated ? mb_substr($body, 0, self::BODY_LIMIT) : $body, + 'truncated' => $truncated, + ]; + }); + } +} diff --git a/src/Chatbot/Mcp/Tool/GetTripTool.php b/src/Chatbot/Mcp/Tool/GetTripTool.php new file mode 100644 index 00000000..46d4b1aa --- /dev/null +++ b/src/Chatbot/Mcp/Tool/GetTripTool.php @@ -0,0 +1,44 @@ +runner->run('get_trip', get_defined_vars(), function () use ($code, $hotelCode): array { + $trip = $this->catalog->get(trim($code), $hotelCode); + + if (null === $trip) { + throw new ToolCallException(sprintf('No trip with code "%s". Use search_trips to find trip codes.', $code)); + } + + return $trip; + }); + } +} diff --git a/src/Chatbot/Mcp/Tool/SearchContentTool.php b/src/Chatbot/Mcp/Tool/SearchContentTool.php new file mode 100644 index 00000000..6e88d002 --- /dev/null +++ b/src/Chatbot/Mcp/Tool/SearchContentTool.php @@ -0,0 +1,47 @@ +runner->run('search_content', get_defined_vars(), function () use ($query, $type, $site, $limit): array { + if ('' === trim($query)) { + throw new ToolCallException('query must not be empty.'); + } + + return ['results' => $this->search->search($query, $type, $site, max(1, min(8, $limit ?? 5)))]; + }); + } +} diff --git a/src/Chatbot/Mcp/Tool/SearchTripsTool.php b/src/Chatbot/Mcp/Tool/SearchTripsTool.php new file mode 100644 index 00000000..fff579e2 --- /dev/null +++ b/src/Chatbot/Mcp/Tool/SearchTripsTool.php @@ -0,0 +1,97 @@ +runner->run('search_trips', get_defined_vars(), function () use ($query, $dateFrom, $dateTo, $country, $region, $concept, $maxPrice, $persons, $daytrip, $site, $limit): array { + $from = $this->date($dateFrom, 'dateFrom'); + $to = $this->date($dateTo, 'dateTo'); + + if (null !== $from && null !== $to && $from > $to) { + throw new ToolCallException('dateFrom must not be after dateTo.'); + } + + return $this->catalog->search(new TripCriteria( + query: $this->text($query), + dateFrom: $from, + dateTo: $to, + country: $this->text($country), + region: $this->text($region), + concept: $this->text($concept), + maxPrice: $maxPrice, + persons: $persons, + daytrip: $daytrip, + site: $site, + limit: max(1, min(10, $limit ?? 5)), + )); + }); + } + + private function date(?string $value, string $name): ?\DateTimeImmutable + { + if (null === $value || '' === trim($value)) { + return null; + } + + $date = \DateTimeImmutable::createFromFormat('!Y-m-d', trim($value)); + + if (false === $date || $date->format('Y-m-d') !== trim($value)) { + throw new ToolCallException(sprintf('%s must be a date in the form YYYY-MM-DD.', $name)); + } + + return $date; + } + + private function text(?string $value): ?string + { + return null === $value || '' === trim($value) ? null : trim($value); + } +} diff --git a/src/Chatbot/Mcp/ToolCallRunner.php b/src/Chatbot/Mcp/ToolCallRunner.php new file mode 100644 index 00000000..d27f9b16 --- /dev/null +++ b/src/Chatbot/Mcp/ToolCallRunner.php @@ -0,0 +1,71 @@ + $arguments as received, for the log + * @param callable(): array $call + * + * @throws ToolCallException invalid input or rate limit; the model sees the message + */ + public function run(string $tool, array $arguments, callable $call): CallToolResult + { + $arguments = array_filter($arguments, static fn (mixed $value): bool => null !== $value); + $limit = $this->chatbotMcpLimiter->create($this->security->getUser()?->getUserIdentifier() ?? 'anonymous')->consume(); + + if (!$limit->isAccepted()) { + $this->chatbotLogger->warning('MCP tool call rate-limited', ['tool' => $tool, 'arguments' => $arguments]); + + throw new ToolCallException(sprintf('Too many tool calls. Retry after %s.', $limit->getRetryAfter()->format('H:i:s'))); + } + + $started = hrtime(true); + + try { + $result = $call(); + } catch (ToolCallException $e) { + $this->chatbotLogger->info('MCP tool call rejected', ['tool' => $tool, 'arguments' => $arguments, 'error' => $e->getMessage()]); + + throw $e; + } catch (\Throwable $e) { + $this->chatbotLogger->error('MCP tool call failed', ['tool' => $tool, 'arguments' => $arguments, 'exception' => $e]); + + throw $e; + } + + $this->chatbotLogger->info('MCP tool call', [ + 'tool' => $tool, + 'arguments' => $arguments, + 'resultCount' => is_array($result['results'] ?? null) ? count($result['results']) : 1, + 'durationMs' => (int) ((hrtime(true) - $started) / 1_000_000), + ]); + + return new CallToolResult([new TextContent(json_encode($result, \JSON_UNESCAPED_SLASHES | \JSON_UNESCAPED_UNICODE | \JSON_THROW_ON_ERROR | \JSON_INVALID_UTF8_SUBSTITUTE))]); + } +} diff --git a/src/Chatbot/Search/ContentSearch.php b/src/Chatbot/Search/ContentSearch.php new file mode 100644 index 00000000..581c1fbc --- /dev/null +++ b/src/Chatbot/Search/ContentSearch.php @@ -0,0 +1,85 @@ + ['page'], + 'product' => ['product'], + 'hotel' => ['hotel'], + 'destination' => ['region', 'country', 'city'], + 'concept' => ['concept'], + 'news' => ['news'], + ]; + + private const TYPE_WEIGHTS = [ + 'product' => 1.4, + 'hotel' => 1.4, + 'region' => 1.2, + 'country' => 1.2, + 'city' => 1.2, + 'concept' => 1.2, + 'page' => 1.0, + 'news' => 0.8, + ]; + + public function __construct(private readonly ContentDocumentRepository $repository) + { + } + + /** + * @param string|null $type a key of TYPES + * + * @return list + */ + public function search(string $query, ?string $type, ?string $site, int $limit): array + { + $booleanQuery = ContentSearchQuery::booleanQuery($query); + + if (null === $booleanQuery) { + return []; + } + + $rows = $this->repository->searchFulltext($booleanQuery, null === $type ? [] : self::TYPES[$type], $site, self::TYPE_WEIGHTS, $limit); + $terms = ContentSearchQuery::terms($query); + + return array_map(static fn (array $row): array => [ + 'type' => self::publicType($row['type']), + 'title' => $row['title'], + 'url' => $row['url'], + 'site' => $row['site'], + 'breadcrumb' => $row['breadcrumb'], + 'excerpt' => Excerpt::around($row['body_md'], $terms), + ], $rows); + } + + /** + * The tool's name for a stored type. + */ + public static function publicType(string $type): string + { + foreach (self::TYPES as $public => $types) { + if (in_array($type, $types, true)) { + return $public; + } + } + + return $type; + } +} diff --git a/src/Chatbot/Search/ContentSearchQuery.php b/src/Chatbot/Search/ContentSearchQuery.php new file mode 100644 index 00000000..b71356f7 --- /dev/null +++ b/src/Chatbot/Search/ContentSearchQuery.php @@ -0,0 +1,66 @@ + + */ + public static function terms(string $text): array + { + $tokens = preg_split('/[^\p{L}\p{N}]+/u', mb_strtolower($text), -1, \PREG_SPLIT_NO_EMPTY) ?: []; + $terms = []; + + foreach ($tokens as $token) { + if (mb_strlen($token) < self::MIN_TOKEN_LENGTH || in_array($token, self::STOPWORDS, true)) { + continue; + } + $terms[$token] = true; + } + + return array_slice(array_keys($terms), 0, self::MAX_TERMS); + } + + /** + * The boolean-mode query, or null when nothing searchable is left. + */ + public static function booleanQuery(string $text): ?string + { + $terms = self::terms($text); + + return [] === $terms ? null : implode(' ', array_map(static fn (string $term): string => $term.'*', $terms)); + } +} diff --git a/src/Chatbot/Search/Excerpt.php b/src/Chatbot/Search/Excerpt.php new file mode 100644 index 00000000..ee741f17 --- /dev/null +++ b/src/Chatbot/Search/Excerpt.php @@ -0,0 +1,53 @@ + $terms lower-cased, matched as prefixes like the full-text search + */ + public static function around(string $text, array $terms, int $length = self::LENGTH): string + { + $text = trim((string) preg_replace('/\s+/u', ' ', $text)); + $textLength = mb_strlen($text); + + if ($textLength <= $length) { + return $text; + } + + $position = null; + $lower = mb_strtolower($text); + foreach ($terms as $term) { + $found = mb_strpos($lower, $term); + if (false !== $found && (null === $position || $found < $position)) { + $position = $found; + } + } + + // Start a third of the window before the match, so it is read in context. + $start = max(0, min(($position ?? 0) - intdiv($length, 3), $textLength - $length)); + $excerpt = mb_substr($text, $start, $length); + + if ($start > 0) { + $space = mb_strpos($excerpt, ' '); + $excerpt = '…'.(false !== $space && $space < 40 ? mb_substr($excerpt, $space + 1) : $excerpt); + } + + if ($start + $length < $textLength) { + $space = mb_strrpos($excerpt, ' '); + $excerpt = (false !== $space && mb_strlen($excerpt) - $space < 40 ? mb_substr($excerpt, 0, $space) : $excerpt).'…'; + } + + return $excerpt; + } +} diff --git a/src/Chatbot/TripCatalog.php b/src/Chatbot/TripCatalog.php new file mode 100644 index 00000000..aaae0557 --- /dev/null +++ b/src/Chatbot/TripCatalog.php @@ -0,0 +1,653 @@ +>, dataAsOf: ?string} + */ + public function search(TripCriteria $criteria): array + { + $catalog = $this->load(); + $products = $this->candidateProducts($catalog, $criteria); + $today = $this->today(); + $results = []; + + foreach ($products as $product) { + $data = $product->getData() ?? []; + $dates = []; + $hotelCodes = []; + + foreach ($catalog['rowsByCode'][$this->normalize($data['code'] ?? null)] ?? [] as $row) { + $summary = $this->summarize($row); + + if (!$this->qualifies($row, $summary, $criteria, $today)) { + continue; + } + + $dateId = $row['date']['busProId']; + $hotelCodes[$this->normalize($row['hotel']['code'])] = true; + $current = $dates[$dateId] ?? null; + + $dates[$dateId] = [ + 'dateFrom' => $row['date']['from'], + 'dateTo' => $row['date']['to'], + 'priceFrom' => null === $current ? $summary['priceFrom'] : $this->minPrice($current['priceFrom'], $summary['priceFrom']), + 'availability' => self::AVAILABILITY_FREE === ($current['availability'] ?? null) ? self::AVAILABILITY_FREE : $summary['availability'], + ]; + } + + if ([] === $dates) { + continue; + } + + usort($dates, static fn (array $a, array $b): int => $a['dateFrom'] <=> $b['dateFrom']); + + $results[] = $this->tripHeader($catalog, $product) + [ + 'hotels' => $this->hotelNames($catalog, array_keys($hotelCodes)), + 'matchingDates' => count($dates), + 'nextDates' => array_slice($dates, 0, self::NEXT_DATES), + ]; + } + + usort($results, static fn (array $a, array $b): int => [$a['nextDates'][0]['dateFrom'], $a['name']] <=> [$b['nextDates'][0]['dateFrom'], $b['name']]); + + return ['results' => array_slice($results, 0, $criteria->limit), 'dataAsOf' => $this->dataAsOf($catalog['export'])]; + } + + /** + * Everything about one trip, or null for a code no published product carries. + * + * @return array|null + */ + public function get(string $code, ?string $hotelCode = null): ?array + { + $catalog = $this->load(); + $products = $catalog['productsByCode'][$this->normalize($code)] ?? []; + + if ([] === $products) { + return null; + } + + usort($products, static fn (ContentDocument $a, ContentDocument $b): int => [null === $a->getUrl(), $a->getUid()] <=> [null === $b->getUrl(), $b->getUid()]); + $product = $products[0]; + $today = $this->today(); + + $rows = array_values(array_filter( + $catalog['rowsByCode'][$this->normalize($code)] ?? [], + fn (array $row): bool => $row['date']['from'] >= $today, + )); + + if (null !== $hotelCode && '' !== trim($hotelCode)) { + $wanted = $this->hotelCodesFor($catalog, $hotelCode); + $rows = array_values(array_filter($rows, fn (array $row): bool => in_array($this->normalize($row['hotel']['code']), $wanted, true))); + } + + usort($rows, static fn (array $a, array $b): int => [$a['date']['from'], $a['hotel']['code']] <=> [$b['date']['from'], $b['hotel']['code']]); + + $hotelCodes = array_values(array_unique(array_map(fn (array $row): string => $this->normalize($row['hotel']['code']), $rows))); + + return $this->tripHeader($catalog, $product) + [ + 'alsoPublishedAt' => array_values(array_map( + static fn (ContentDocument $other): array => ['site' => $other->getSite(), 'url' => $other->getUrl()], + array_filter(array_slice($products, 1), static fn (ContentDocument $other): bool => null !== $other->getUrl()), + )), + 'description' => $this->truncate($product->getBodyMd(), self::BODY_LIMIT), + 'hotels' => $this->hotelDetails($catalog, $hotelCodes, $product), + 'dates' => $this->dates($catalog, $rows), + 'dataAsOf' => $this->dataAsOf($catalog['export']), + ]; + } + + /** + * Logs the codes the join cannot place, on both sides. The same gaps the website's + * product import tolerates, worth a periodic look. + */ + public function logUnmatchedCodes(): void + { + $catalog = $this->load(); + + $exportProducts = array_keys($catalog['rowsByCode']); + $contentProducts = array_keys($catalog['productsByCode']); + + $exportHotels = []; + foreach ($catalog['export']->rows as $row) { + $exportHotels[$this->normalize($row['hotel']['code'] ?? null)] = true; + } + unset($exportHotels['']); + + $ambiguous = []; + foreach ($catalog['hotelsByCode'] as $hotelCode => $hotels) { + if (count($hotels) > 1) { + $ambiguous[$hotelCode] = array_map(static fn (ContentDocument $hotel): int => $hotel->getUid(), $hotels); + } + } + + $context = [ + 'exportProductsWithoutContent' => array_values(array_diff($exportProducts, $contentProducts)), + 'contentProductsWithoutDates' => array_values(array_diff($contentProducts, $exportProducts)), + 'exportHotelsWithoutContent' => array_values(array_diff(array_keys($exportHotels), array_keys($catalog['hotelsByCode']))), + 'ambiguousHotelCodes' => $ambiguous, + ]; + + $this->logger->info('Content feed: unmatched codes', $context + [ + 'counts' => array_map('count', $context), + ]); + } + + /** + * @return array{ + * export: Export, + * records: array, + * productsByCode: array>, + * hotelsByCode: array>, + * rowsByCode: array>>, + * } + */ + private function load(): array + { + $export = $this->exportBuilder->build(); + $records = $this->repository->findRecords(self::RECORD_TYPES, self::LANGUAGE); + + $productsByCode = []; + $hotelsByCode = []; + + foreach ($records as $record) { + $data = $record->getData() ?? []; + + if ('product' === $record->getType() && '' !== ($code = $this->normalize($data['code'] ?? null))) { + $productsByCode[$code][] = $record; + } + + if ('hotel' === $record->getType()) { + $codes = is_array($data['codes'] ?? null) ? $data['codes'] : []; + $codes[] = $data['code'] ?? null; + + foreach (array_unique(array_map($this->normalize(...), $codes)) as $code) { + if ('' !== $code) { + $hotelsByCode[$code][] = $record; + } + } + } + } + + // A code that is one hotel's own and another's alias resolves to the owner. + foreach ($hotelsByCode as $code => $hotels) { + usort($hotels, fn (ContentDocument $a, ContentDocument $b): int => [$this->normalize($a->getData()['code'] ?? null) !== $code, $a->getUid()] <=> [$this->normalize($b->getData()['code'] ?? null) !== $code, $b->getUid()]); + $hotelsByCode[$code] = $hotels; + } + + $rowsByCode = []; + foreach ($export->rows as $row) { + $code = $this->normalize($row['product']['code'] ?? null); + if ('' !== $code) { + $rowsByCode[$code][] = $row; + } + } + + return [ + 'export' => $export, + 'records' => $records, + 'productsByCode' => $productsByCode, + 'hotelsByCode' => $hotelsByCode, + 'rowsByCode' => $rowsByCode, + ]; + } + + /** + * Published products narrowed by everything the content index can resolve: site, day + * trip, concept, destination and free text. + * + * @param array $catalog + * + * @return list + */ + private function candidateProducts(array $catalog, TripCriteria $criteria): array + { + /** @var array $records */ + $records = $catalog['records']; + + $conceptUids = null === $criteria->concept ? null : $this->matchingUids($records, ['concept'], $criteria->concept, matchCode: true); + $countryUids = null === $criteria->country ? null : $this->matchingUids($records, ['country'], $criteria->country, matchCode: true); + $regionUids = null === $criteria->region ? null : $this->matchingUids($records, ['region'], $criteria->region); + $cityUids = null === $criteria->region ? null : $this->matchingUids($records, ['city'], $criteria->region); + $queryUids = null === $criteria->query ? null : $this->productUidsMatchingText($records, $criteria->query); + + $products = []; + + foreach ($records as $record) { + if ('product' !== $record->getType() || null === $record->getUrl()) { + continue; + } + + $data = $record->getData() ?? []; + + // Closed group trips and staff trainings are not offered; get() still answers + // someone who names their code. + if (false === ($data['bookable'] ?? null) + || $record->isHiddenFromSearch() + || (null !== $criteria->site && $record->getSite() !== $criteria->site) + || (null !== $criteria->daytrip && (bool) ($data['daytrip'] ?? false) !== $criteria->daytrip) + || (null !== $conceptUids && !in_array($data['conceptUid'] ?? null, $conceptUids, true)) + || (null !== $countryUids && !in_array($data['countryUid'] ?? null, $countryUids, true)) + || (null !== $queryUids && !in_array($record->getUid(), $queryUids, true))) { + continue; + } + + if (null !== $regionUids) { + $productRegions = array_merge([$data['regionUid'] ?? null], is_array($data['additionalRegionUids'] ?? null) ? $data['additionalRegionUids'] : []); + + if ([] === array_intersect($productRegions, $regionUids) && !in_array($data['cityUid'] ?? null, $cityUids ?? [], true)) { + continue; + } + } + + $products[] = $record; + } + + return $products; + } + + /** + * Uids of the records of the given types whose name contains the text, or whose code + * equals it. + * + * @param array $records + * @param list $types + * + * @return list + */ + private function matchingUids(array $records, array $types, string $text, bool $matchCode = false): array + { + $needle = mb_strtolower(trim($text)); + $uids = []; + + foreach ($records as $record) { + if (!in_array($record->getType(), $types, true)) { + continue; + } + + $name = mb_strtolower($record->getTitle()); + $code = $this->normalize($record->getData()['code'] ?? null); + + if (('' !== $name && (str_contains($name, $needle) || str_contains($needle, $name))) + || ($matchCode && '' !== $code && $code === $this->normalize($text))) { + $uids[] = $record->getUid(); + } + } + + return $uids; + } + + /** + * Products reached by a full-text match on themselves or on a hotel, destination or + * concept they belong to. + * + * @param array $records + * + * @return list + */ + private function productUidsMatchingText(array $records, string $text): array + { + $booleanQuery = ContentSearchQuery::booleanQuery($text); + + if (null === $booleanQuery) { + return array_values(array_map(static fn (ContentDocument $record): int => $record->getUid(), array_filter($records, static fn (ContentDocument $record): bool => 'product' === $record->getType()))); + } + + $matched = []; + foreach ($this->repository->matchRecords($booleanQuery, self::RECORD_TYPES, self::LANGUAGE) as $hit) { + $matched[$hit['type']][$hit['uid']] = true; + } + + $uids = []; + + foreach ($records as $record) { + if ('product' !== $record->getType()) { + continue; + } + + $data = $record->getData() ?? []; + $hotelUids = is_array($data['hotelUids'] ?? null) ? $data['hotelUids'] : []; + $regionUids = array_merge([$data['regionUid'] ?? null], is_array($data['additionalRegionUids'] ?? null) ? $data['additionalRegionUids'] : []); + + $hit = isset($matched['product'][$record->getUid()]) + || [] !== array_intersect($hotelUids, array_keys($matched['hotel'] ?? [])) + || [] !== array_intersect($regionUids, array_keys($matched['region'] ?? [])) + || isset($matched['country'][$data['countryUid'] ?? -1]) + || isset($matched['city'][$data['cityUid'] ?? -1]) + || isset($matched['concept'][$data['conceptUid'] ?? -1]); + + // Hotels also name their products, for products whose hotelUids miss them. + if (!$hit) { + foreach (array_keys($matched['hotel'] ?? []) as $hotelUid) { + $hotelProducts = ($records['hotel:'.$hotelUid] ?? null)?->getData()['productUids'] ?? []; + if (is_array($hotelProducts) && in_array($record->getUid(), $hotelProducts, true)) { + $hit = true; + break; + } + } + } + + if ($hit) { + $uids[] = $record->getUid(); + } + } + + return $uids; + } + + /** + * @param array $row + * + * @return array{priceFrom: ?float, places: int, availability: ?string} + */ + private function summarize(array $row): array + { + $priceFrom = null; + $places = 0; + $availability = null; + + foreach ($row['rooms'] ?? [] as $room) { + if (self::STATUS_BOOKING_STOP === ($room['status'] ?? null)) { + continue; + } + + $places += (int) ($room['available'] ?? 0) * (int) ($room['maxPax'] ?? 0); + $availability = self::STATUS_FREE === ($room['status'] ?? null) || self::AVAILABILITY_FREE === $availability + ? self::AVAILABILITY_FREE + : self::AVAILABILITY_ON_REQUEST; + + if (is_int($room['price'] ?? null) || is_float($room['price'] ?? null)) { + $priceFrom = $this->minPrice($priceFrom, (float) $room['price']); + } + } + + return ['priceFrom' => $priceFrom, 'places' => $places, 'availability' => $availability]; + } + + /** + * @param array $row + * @param array{priceFrom: ?float, places: int, availability: ?string} $summary + */ + private function qualifies(array $row, array $summary, TripCriteria $criteria, string $today): bool + { + $from = $row['date']['from']; + $to = $row['date']['to']; + + return $from >= $today + && null !== $summary['availability'] + && (null === $criteria->dateFrom || $to >= $criteria->dateFrom->format('Y-m-d')) + && (null === $criteria->dateTo || $from <= $criteria->dateTo->format('Y-m-d')) + && (null === $criteria->maxPrice || (null !== $summary['priceFrom'] && $summary['priceFrom'] <= $criteria->maxPrice)) + && (null === $criteria->persons || $summary['places'] >= $criteria->persons); + } + + /** + * @param array $catalog + * + * @return array + */ + private function tripHeader(array $catalog, ContentDocument $product): array + { + $data = $product->getData() ?? []; + $name = fn (string $type, mixed $uid): ?string => is_int($uid) ? ($catalog['records'][$type.':'.$uid] ?? null)?->getTitle() : null; + + return [ + 'code' => $data['code'] ?? null, + 'name' => $product->getTitle(), + 'url' => $product->getUrl(), + 'site' => $product->getSite(), + 'destination' => array_filter([ + 'country' => $name('country', $data['countryUid'] ?? null), + 'region' => $name('region', $data['regionUid'] ?? null), + 'city' => $name('city', $data['cityUid'] ?? null), + ]), + 'concept' => $name('concept', $data['conceptUid'] ?? null), + 'daytrip' => (bool) ($data['daytrip'] ?? false), + ]; + } + + /** + * @param array $catalog + * @param list $hotelCodes normalized + * + * @return list + */ + private function hotelNames(array $catalog, array $hotelCodes): array + { + $names = []; + foreach ($hotelCodes as $code) { + $hotel = $catalog['hotelsByCode'][$code][0] ?? null; + if (null !== $hotel && '' !== $hotel->getTitle()) { + $names[$hotel->getTitle()] = true; + } + } + + return array_keys($names); + } + + /** + * @param array $catalog + * @param list $hotelCodes normalized codes of the hotels with dates + * + * @return list> + */ + private function hotelDetails(array $catalog, array $hotelCodes, ContentDocument $product): array + { + $hotels = []; + + foreach ($hotelCodes as $code) { + $hotel = $catalog['hotelsByCode'][$code][0] ?? null; + if (null !== $hotel) { + $hotels[$hotel->getUid()] = $hotel; + } + } + + // A trip without upcoming dates still describes its hotels. + if ([] === $hotels) { + foreach ((array) ($product->getData()['hotelUids'] ?? []) as $uid) { + $hotel = is_int($uid) ? ($catalog['records']['hotel:'.$uid] ?? null) : null; + if (null !== $hotel) { + $hotels[$hotel->getUid()] = $hotel; + } + } + } + + return array_values(array_map(fn (ContentDocument $hotel): array => [ + 'code' => $hotel->getData()['code'] ?? null, + 'name' => $hotel->getTitle(), + 'url' => $hotel->getUrl(), + 'facts' => is_array($hotel->getData()['facts'] ?? null) ? array_filter($hotel->getData()['facts'], static fn (mixed $value): bool => null !== $value && '' !== $value) : [], + 'description' => $this->truncate($hotel->getBodyMd(), self::HOTEL_BODY_LIMIT), + ], $hotels)); + } + + /** + * Upcoming dates with their bookable rooms per hotel. Pickups for the first few only: + * they rarely differ between dates and would otherwise dominate the answer. + * + * @param array $catalog + * @param list> $rows sorted by date + * + * @return list> + */ + private function dates(array $catalog, array $rows): array + { + $dates = []; + + foreach ($rows as $row) { + $dateId = $row['date']['busProId']; + $hotel = $catalog['hotelsByCode'][$this->normalize($row['hotel']['code'])][0] ?? null; + + $rooms = []; + foreach ($row['rooms'] ?? [] as $room) { + if (self::STATUS_BOOKING_STOP === ($room['status'] ?? null)) { + continue; + } + + $rooms[] = array_filter([ + 'room' => $room['label'] ?? null, + 'board' => $room['boardCode'] ?? null, + 'nights' => $room['nights'] ?? null, + 'price' => $room['price'] ?? null, + 'availability' => self::STATUS_FREE === ($room['status'] ?? null) ? self::AVAILABILITY_FREE : self::AVAILABILITY_ON_REQUEST, + ], static fn (mixed $value): bool => null !== $value); + } + + $dates[$dateId] ??= [ + 'dateFrom' => $row['date']['from'], + 'dateTo' => $row['date']['to'], + 'hotels' => [], + 'pickups' => $this->pickups($row), + ]; + + $dates[$dateId]['hotels'][] = [ + 'hotel' => $hotel?->getTitle() ?? $row['hotel']['code'], + 'hotelCode' => $row['hotel']['code'], + 'rooms' => $rooms, + 'bookable' => [] !== $rooms, + ]; + } + + $dates = array_values($dates); + + foreach ($dates as $i => $date) { + if ($i >= self::DATES_WITH_PICKUPS) { + unset($dates[$i]['pickups']); + } + } + + return $dates; + } + + /** + * @param array $row + * + * @return list> + */ + private function pickups(array $row): array + { + $pickups = []; + + foreach ($row['pickups'] ?? [] as $pickup) { + if (0 === ($pickup['busProId'] ?? 0) || null === ($pickup['city'] ?? null)) { + continue; + } + + $pickups[] = array_filter([ + 'city' => $pickup['city'], + 'place' => $pickup['street'] ?? null, + // A clock reading without an offset, as BusPro states it. + 'departure' => is_string($pickup['datetime'] ?? null) ? str_replace('T', ' ', substr($pickup['datetime'], 0, 16)) : null, + 'surcharge' => $pickup['price'] ?? null, + ], static fn (mixed $value): bool => null !== $value); + } + + return $pickups; + } + + /** + * Normalized codes of the hotel a code names: all its own codes when TYPO3 knows the + * hotel, else just the code. + * + * @param array $catalog + * + * @return list + */ + private function hotelCodesFor(array $catalog, string $hotelCode): array + { + $hotel = $catalog['hotelsByCode'][$this->normalize($hotelCode)][0] ?? null; + + if (null === $hotel) { + return [$this->normalize($hotelCode)]; + } + + $codes = is_array($hotel->getData()['codes'] ?? null) ? $hotel->getData()['codes'] : []; + $codes[] = $hotel->getData()['code'] ?? null; + + return array_values(array_filter(array_unique(array_map($this->normalize(...), $codes)))); + } + + private function dataAsOf(Export $export): string + { + return ($export->sourceGeneratedAt ?? $export->generatedAt)->format('c'); + } + + private function truncate(string $text, int $limit): string + { + if (mb_strlen($text) <= $limit) { + return $text; + } + + return mb_substr($text, 0, $limit)."\n\n[… gekürzt, vollständiger Text unter der URL]"; + } + + private function minPrice(?float $a, ?float $b): ?float + { + if (null === $a || null === $b) { + return $a ?? $b; + } + + return min($a, $b); + } + + private function today(): string + { + return $this->clock->now()->setTimezone(new \DateTimeZone('Europe/Berlin'))->format('Y-m-d'); + } + + private function normalize(mixed $code): string + { + return is_string($code) ? strtoupper(trim($code)) : ''; + } +} diff --git a/src/Command/ContentCheckFreshnessCommand.php b/src/Command/ContentCheckFreshnessCommand.php new file mode 100644 index 00000000..d652be99 --- /dev/null +++ b/src/Command/ContentCheckFreshnessCommand.php @@ -0,0 +1,63 @@ +addOption('max-age', null, InputOption::VALUE_REQUIRED, 'Maximum age of the last good manifest, in minutes', '60'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $state = $this->stateRepository->get(); + $generatedAt = $state->getManifestGeneratedAt(); + $maxAge = (int) $input->getOption('max-age'); + + if (null === $generatedAt) { + $io->error('No content manifest has been accepted yet.'); + + return Command::FAILURE; + } + + $ageMinutes = (int) floor(($this->clock->now()->getTimestamp() - $generatedAt->getTimestamp()) / 60); + + if ($ageMinutes > $maxAge) { + $io->error(sprintf('The last good content manifest is %d minutes old (generated %s). Last run error: %s', $ageMinutes, $generatedAt->format('c'), $state->getLastError() ?? 'none')); + + return Command::FAILURE; + } + + $io->success(sprintf('The last good content manifest is %d minutes old.', $ageMinutes)); + + return Command::SUCCESS; + } +} diff --git a/src/Command/ContentSyncCommand.php b/src/Command/ContentSyncCommand.php new file mode 100644 index 00000000..7efc7468 --- /dev/null +++ b/src/Command/ContentSyncCommand.php @@ -0,0 +1,83 @@ +addOption('refresh', null, InputOption::VALUE_NONE, sprintf('Also refetch items stored more than %d days ago, oldest first', ContentSyncer::REFRESH_AFTER_DAYS)) + ->addOption('ignore-window', null, InputOption::VALUE_NONE, sprintf('Fetch every due page, not just %d, outside the night window', ContentSyncer::PAGES_OUTSIDE_WINDOW)) + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + if (!$this->lock()) { + $io->warning('Another content sync is running.'); + + return Command::SUCCESS; + } + + try { + $report = $this->syncer->run((bool) $input->getOption('refresh'), (bool) $input->getOption('ignore-window')); + } finally { + $this->release(); + } + + $io->definitionList( + ['Added' => $report->added], + ['Deleted' => $report->deleted], + ['Records fetched' => $report->records], + ['Pages fetched' => $report->pages], + ['Failed' => $report->failed], + ['Pages deferred' => $report->deferred], + ); + + if (null !== $report->pagePhaseEndedBy) { + $io->warning('Page phase ended early: '.$report->pagePhaseEndedBy); + } + + if (null !== $report->stoppedBy) { + $io->error('Run stopped: '.$report->stoppedBy); + + return Command::FAILURE; + } + + $io->success('Content feed synced.'); + + return Command::SUCCESS; + } +} diff --git a/src/ContentFeed/ContentFeedClient.php b/src/ContentFeed/ContentFeedClient.php new file mode 100644 index 00000000..66f2c796 --- /dev/null +++ b/src/ContentFeed/ContentFeedClient.php @@ -0,0 +1,312 @@ + */ + private array $siteHosts = []; + + public function __construct( + private readonly HttpClientInterface $httpClient, + private readonly ClockInterface $clock, + private readonly LoggerInterface $logger, + private readonly string $baseUrl, + private readonly string $apiKey, + ) { + } + + public function startRun(): void + { + $this->consecutiveFailures = 0; + $this->consecutiveSlow = 0; + $this->notBefore = null; + $this->lastPageAt = null; + $this->siteHosts = []; + } + + /** + * The whole manifest. Anything but a usable one stops the run: an empty TYPO3 is a bug, + * not a deletion. + * + * @throws ContentFeedStoppedException + */ + public function manifest(): ContentManifest + { + if ('' === $this->apiKey) { + throw new ContentFeedStoppedException('APP_CMS_API_KEY is not configured'); + } + + $response = $this->send($this->apiUrl('api/content/manifest'), []); + + if (null === $response || 200 !== $response['status']) { + throw new ContentFeedStoppedException(sprintf('Manifest request failed (%s)', $response['status'] ?? 'timeout')); + } + + try { + $data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new ContentFeedStoppedException('Manifest is not valid JSON', 0, $e); + } + + if (!is_array($data) || !is_string($data['generatedAt'] ?? null) || !is_array($data['items'] ?? null) || [] === $data['items']) { + throw new ContentFeedStoppedException('Manifest has no generatedAt or no items'); + } + + $generatedAt = ManifestItem::localTime($data['generatedAt']); + if (null === $generatedAt) { + throw new ContentFeedStoppedException('Manifest generatedAt is unparseable'); + } + + $items = []; + foreach ($data['items'] as $raw) { + try { + $item = ManifestItem::fromArray(is_array($raw) ? $raw : []); + } catch (\InvalidArgumentException $e) { + // A malformed entry would otherwise read as a deletion of whatever it was. + throw new ContentFeedStoppedException('Manifest contains an unusable item: '.$e->getMessage(), 0, $e); + } + $items[$item->key()] = $item; + } + + $hosts = []; + foreach (is_array($data['sites'] ?? null) ? $data['sites'] : [] as $site) { + $host = is_array($site) && is_string($site['base'] ?? null) ? parse_url($site['base'], \PHP_URL_HOST) : null; + if (is_string($host)) { + $hosts[] = strtolower($host); + } + } + $this->siteHosts = array_values(array_unique($hosts)); + + return new ContentManifest($generatedAt, $items, $this->siteHosts); + } + + /** + * Records of one type by uid, keyed by uid. A requested uid that is absent is no + * longer eligible. + * + * @param list $uids at most RECORDS_BATCH_SIZE + * + * @return array> + * + * @throws ContentRequestFailedException this batch failed, the run goes on + * @throws ContentFeedStoppedException + */ + public function records(string $type, array $uids): array + { + if (count($uids) > self::RECORDS_BATCH_SIZE) { + throw new \InvalidArgumentException(sprintf('At most %d uids per records request', self::RECORDS_BATCH_SIZE)); + } + + $response = $this->send($this->apiUrl('api/content/records'), ['type' => $type, 'uids' => implode(',', $uids)]); + + if (null === $response) { + throw new ContentRequestFailedException(sprintf('Records request for %s timed out', $type)); + } + + if (200 !== $response['status']) { + if ($response['status'] >= 500) { + throw new ContentRequestFailedException(sprintf('Records request for %s failed (%d)', $type, $response['status'])); + } + + // 403 and the like: misconfiguration, which the next request would hit too. + throw new ContentFeedStoppedException(sprintf('Records request for %s answered %d', $type, $response['status'])); + } + + try { + $data = json_decode($response['body'], true, flags: \JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new ContentRequestFailedException(sprintf('Records response for %s is not valid JSON', $type), 0, $e); + } + + // Anything but the documented envelope fails the batch: read as an empty list, it + // would delete every record requested. + if (!is_array($data) || ($data['type'] ?? null) !== $type || !is_array($data['items'] ?? null) || !array_is_list($data['items'])) { + throw new ContentRequestFailedException(sprintf('Records response for %s has no items list', $type)); + } + + $records = []; + foreach ($data['items'] as $record) { + if (is_array($record) && is_int($record['uid'] ?? null)) { + $records[$record['uid']] = $record; + } + } + + return $records; + } + + /** + * The bare HTML fragment of one page, fetched at most once a second. + * + * @throws PagePhaseEndedException + * @throws ContentFeedStoppedException + */ + public function page(string $contentUrl): PageFetchResult + { + $host = parse_url($contentUrl, \PHP_URL_HOST); + + // The key goes with every request, so it only ever goes to a host of the feed's own sites. + if (!is_string($host) || !in_array(strtolower($host), $this->siteHosts, true)) { + $this->logger->warning('Content feed: contentUrl outside the manifest sites, skipped', ['contentUrl' => $contentUrl]); + + return PageFetchResult::failed(0); + } + + if (null !== $this->lastPageAt) { + $this->waitUntil($this->lastPageAt->modify(sprintf('+%d ms', (int) (self::PAGE_INTERVAL_SECONDS * 1000)))); + } + $this->lastPageAt = $this->clock->now(); + + $response = $this->send($contentUrl, [], countSlow: true); + + if (null === $response) { + return PageFetchResult::transient(); + } + + if (200 === $response['status']) { + return PageFetchResult::ok($response['body']); + } + + return PageFetchResult::failed($response['status']); + } + + /** + * Sends one GET under the load rules. Null on a timeout or transport error. + * + * @param array $query + * + * @return array{status: int, body: string}|null + * + * @throws ContentFeedStoppedException + * @throws PagePhaseEndedException + */ + private function send(string $url, array $query, bool $countSlow = false): ?array + { + if (null !== $this->notBefore) { + $this->waitUntil($this->notBefore); + $this->notBefore = null; + } + + $result = null; + $duration = 0.0; + + try { + $response = $this->httpClient->request('GET', $url, [ + 'query' => $query, + 'headers' => ['X-Api-Key' => $this->apiKey], + 'timeout' => self::TIMEOUT_SECONDS, + 'max_duration' => self::TIMEOUT_SECONDS, + ]); + $result = ['status' => $response->getStatusCode(), 'body' => $response->getContent(false)]; + $duration = $this->duration($response); + } catch (TransportExceptionInterface $e) { + $this->logger->warning('Content feed: request failed', ['url' => $url, 'error' => $e->getMessage()]); + $duration = self::TIMEOUT_SECONDS; + } + + if (null !== $result && 503 === $result['status']) { + throw new ContentFeedUnavailableException($this->retryAfter($response)); + } + + if (null === $result || $result['status'] >= 500) { + if (++$this->consecutiveFailures >= self::MAX_CONSECUTIVE_FAILURES) { + throw new ContentFeedStoppedException(sprintf('%d consecutive 5xx responses or timeouts', self::MAX_CONSECUTIVE_FAILURES)); + } + } else { + $this->consecutiveFailures = 0; + } + + if ($duration > self::SLOW_SECONDS) { + $this->notBefore = $this->clock->now()->modify(sprintf('+%d ms', (int) (self::SLOW_PAUSE_SECONDS * 1000))); + + if ($countSlow && ++$this->consecutiveSlow >= self::MAX_CONSECUTIVE_SLOW) { + $this->consecutiveSlow = 0; + throw new PagePhaseEndedException(sprintf('%d consecutive responses slower than %d s', self::MAX_CONSECUTIVE_SLOW, (int) self::SLOW_SECONDS)); + } + } elseif ($countSlow) { + $this->consecutiveSlow = 0; + } + + return $result; + } + + private function duration(ResponseInterface $response): float + { + $total = $response->getInfo('total_time'); + + return is_float($total) || is_int($total) ? (float) $total : 0.0; + } + + private function retryAfter(ResponseInterface $response): ?\DateTimeImmutable + { + $value = $response->getHeaders(false)['retry-after'][0] ?? null; + + if (null === $value || '' === trim($value)) { + return null; + } + + if (ctype_digit(trim($value))) { + return $this->clock->now()->modify(sprintf('+%d seconds', (int) trim($value))); + } + + $date = date_create_immutable($value); + + return false === $date ? null : $date; + } + + private function waitUntil(\DateTimeImmutable $moment): void + { + $seconds = (float) $moment->format('U.u') - (float) $this->clock->now()->format('U.u'); + + if ($seconds > 0) { + $this->clock->sleep($seconds); + } + } + + private function apiUrl(string $path): string + { + return rtrim($this->baseUrl, '/').'/'.$path; + } +} diff --git a/src/ContentFeed/ContentSyncer.php b/src/ContentFeed/ContentSyncer.php new file mode 100644 index 00000000..2441a417 --- /dev/null +++ b/src/ContentFeed/ContentSyncer.php @@ -0,0 +1,367 @@ +stateRepository->get(); + $this->entityManager->persist($state); + $now = $this->clock->now(); + + if (null !== $state->getPausedUntil() && $now < $state->getPausedUntil()) { + $report->stoppedBy = 'Feed switched off by TYPO3 until '.$state->getPausedUntil()->format('c'); + + return $report; + } + + $state->setPausedUntil(null)->setLastRunAt($now); + $this->client->startRun(); + + try { + $manifest = $this->client->manifest(); + $documents = $this->applyManifest($manifest, $report); + $state->setManifestGeneratedAt($manifest->generatedAt); + $this->entityManager->flush(); + + $staleBefore = $refresh ? $now->modify(sprintf('-%d days', self::REFRESH_AFTER_DAYS)) : null; + $this->syncRecords($documents, $staleBefore, $report); + $this->syncPages($documents, $staleBefore, $ignoreWindow || $this->inNightWindow($now), $report); + } catch (ContentFeedUnavailableException $e) { + $state->setPausedUntil($e->retryAfter); + $report->stoppedBy = $e->getMessage(); + } catch (ContentFeedStoppedException $e) { + $report->stoppedBy = $e->getMessage(); + } + + if (isset($documents)) { + $this->merge(); + } + + $state->setLastError($report->stoppedBy ?? $report->pagePhaseEndedBy); + $this->entityManager->flush(); + + if (null === $report->stoppedBy) { + $this->logger->info('Content feed sync finished', $report->toLogContext()); + } else { + $this->logger->warning('Content feed sync stopped', $report->toLogContext()); + } + + if ($report->isComplete() && $report->changedAnything()) { + $this->tripCatalog->logUnmatchedCodes(); + } + + return $report; + } + + /** + * Deletes what the manifest no longer lists and brings every listed item's metadata up + * to date. Content is not touched here. + * + * @return array + */ + private function applyManifest(ContentManifest $manifest, ContentSyncReport $report): array + { + $stored = $this->documentRepository->findAllByKey(); + + foreach (array_diff_key($stored, $manifest->items) as $document) { + $this->entityManager->remove($document); + ++$report->deleted; + } + + $documents = []; + + foreach ($manifest->items as $key => $item) { + $document = $stored[$key] ?? null; + + if (null === $document) { + $document = new ContentDocument($item->type, $item->uid, $item->language, $item->modified); + $this->entityManager->persist($document); + ++$report->added; + } + + $document->setModified($item->modified)->setSite($item->site)->setUrl($item->url); + + if ($document->isPage()) { + $document + ->setTitle($item->title ?? '') + ->setBreadcrumb([] === $item->breadcrumb ? null : implode(' › ', $item->breadcrumb)) + ->setData(['contentUrl' => $item->contentUrl, 'description' => $item->description, 'detailOf' => $item->detailOf]); + } + + $documents[$key] = ['document' => $document, 'item' => $item]; + } + + return $documents; + } + + /** + * @param array $documents + */ + private function syncRecords(array $documents, ?\DateTimeImmutable $staleBefore, ContentSyncReport $report): void + { + /** @var array>> $stale type => uid => documents; the records endpoint takes no language */ + $stale = []; + + foreach ($this->dueForFetch($documents, $staleBefore, pages: false) as $document) { + $stale[$document->getType()][$document->getUid()][] = $document; + } + + foreach ($stale as $type => $byUid) { + foreach (array_chunk(array_keys($byUid), ContentFeedClient::RECORDS_BATCH_SIZE) as $uids) { + try { + $records = $this->client->records($type, $uids); + } catch (ContentRequestFailedException $e) { + $this->logger->warning('Content feed: records batch failed, retried next run', ['type' => $type, 'error' => $e->getMessage()]); + $report->failed += count($uids); + continue; + } + + foreach ($uids as $uid) { + foreach ($byUid[$uid] as $document) { + $record = $records[$uid] ?? null; + + if (null === $record) { + // Requested but absent: no longer eligible. + $this->entityManager->remove($document); + ++$report->deleted; + continue; + } + + $this->applyRecord($document, $record); + ++$report->records; + } + } + + $this->entityManager->flush(); + } + } + } + + /** + * @param array $record + */ + private function applyRecord(ContentDocument $document, array $record): void + { + $title = $this->bodyBuilder->title($document->getType(), $record); + + $document + ->setData($record) + ->setTitle('' !== $title ? $title : $document->getTitle()) + // The merged pages are appended in merge(). + ->setBodyMd($this->bodyBuilder->build($document->getType(), $record)) + ->markFetched(ContentFetchState::Ok, $this->clock->now()); + } + + /** + * @param array $documents + * + * @throws ContentFeedStoppedException + */ + private function syncPages(array $documents, ?\DateTimeImmutable $staleBefore, bool $unlimited, ContentSyncReport $report): void + { + $due = $this->dueForFetch($documents, $staleBefore, pages: true); + + if (!$unlimited && count($due) > self::PAGES_OUTSIDE_WINDOW) { + $report->deferred = count($due) - self::PAGES_OUTSIDE_WINDOW; + $due = array_slice($due, 0, self::PAGES_OUTSIDE_WINDOW); + } + + $fetched = 0; + + try { + foreach ($due as $document) { + $this->fetchPage($document, $report); + + if (0 === ++$fetched % self::FLUSH_EVERY_PAGES) { + $this->entityManager->flush(); + } + } + } catch (PagePhaseEndedException $e) { + $report->pagePhaseEndedBy = $e->getMessage(); + $this->logger->warning('Content feed: page phase ended early', ['reason' => $e->getMessage()]); + } finally { + $this->entityManager->flush(); + } + } + + private function fetchPage(ContentDocument $document, ContentSyncReport $report): void + { + $contentUrl = $document->getData()['contentUrl'] ?? null; + + if (!is_string($contentUrl)) { + $document->markFetched(ContentFetchState::Failed, $this->clock->now()); + ++$report->failed; + + return; + } + + $result = $this->client->page($contentUrl); + + if ($result->transient) { + ++$report->failed; + + return; + } + + if (!$result->isOk()) { + $this->logger->notice('Content feed: page failed, not refetched until it changes', [ + 'uid' => $document->getUid(), + 'contentUrl' => $contentUrl, + 'status' => $result->failedStatus, + ]); + $document->markFetched(ContentFetchState::Failed, $this->clock->now()); + ++$report->failed; + + return; + } + + $markdown = $this->converter->convert($result->html, $document->getUrl() ?? $contentUrl); + $document->setBodyMd($markdown)->markFetched('' === $markdown ? ContentFetchState::Empty : ContentFetchState::Ok, $this->clock->now()); + ++$report->pages; + } + + /** + * Items whose `modified` changed since their last fetch, plus, on a refresh, those + * fetched before `$staleBefore`, oldest first. + * + * @param array $documents + * + * @return list + */ + private function dueForFetch(array $documents, ?\DateTimeImmutable $staleBefore, bool $pages): array + { + $due = []; + + foreach ($documents as ['document' => $document]) { + if ($document->isPage() !== $pages || !$this->entityManager->contains($document)) { + continue; + } + + $outdated = null !== $staleBefore && (null === $document->getSyncedAt() || $document->getSyncedAt() < $staleBefore); + + if ($document->isStale() || $outdated) { + $due[] = $document; + } + } + + usort($due, static fn (ContentDocument $a, ContentDocument $b): int => ($a->getSyncedAt()?->getTimestamp() ?? 0) <=> ($b->getSyncedAt()?->getTimestamp() ?? 0)); + + return $due; + } + + /** + * Appends every detail page to the record it is the detail page of, and keeps the page + * itself out of the index. Recomputed from scratch on every run, so a change or deletion + * on either side re-merges. + */ + private function merge(): void + { + $documents = $this->documentRepository->findAllByKey(); + /** @var array> $pagesByRecord */ + $pagesByRecord = []; + + foreach ($documents as $page) { + if (!$page->isPage()) { + continue; + } + + $target = null; + foreach ($page->getData()['detailOf'] ?? [] as $ref) { + $candidate = $documents[ContentDocument::key($ref['type'], $ref['uid'], $page->getLanguage())] ?? null; + + if (null !== $candidate && ContentFetchState::Ok === $candidate->getFetchState()) { + $target = $candidate; + break; + } + } + + $page->setMergedInto($target?->getRecordRef()); + + if (null !== $target && ContentFetchState::Ok === $page->getFetchState() && '' !== $page->getBodyMd()) { + $pagesByRecord[$target->getKey()][] = $page; + } + } + + foreach ($documents as $record) { + if ($record->isPage() || ContentFetchState::Ok !== $record->getFetchState() || null === $record->getData()) { + continue; + } + + $pages = $pagesByRecord[$record->getKey()] ?? []; + usort($pages, static fn (ContentDocument $a, ContentDocument $b): int => $a->getUid() <=> $b->getUid()); + + $body = $this->bodyBuilder->build($record->getType(), $record->getData()); + foreach ($pages as $page) { + $body .= "\n\n".$page->getBodyMd(); + } + + $record->setBodyMd(trim($body)); + + // A record without a reachable page of its own is linkable through its detail page. + if (null === $record->getUrl() && [] !== $pages) { + $record->setUrl($pages[0]->getUrl())->setSite($record->getSite() ?? $pages[0]->getSite()); + } + } + } + + private function inNightWindow(\DateTimeImmutable $now): bool + { + $hour = (int) $now->setTimezone(new \DateTimeZone(self::NIGHT_WINDOW_TIMEZONE))->format('G'); + + return $hour >= self::NIGHT_WINDOW_FROM_HOUR && $hour < self::NIGHT_WINDOW_TO_HOUR; + } +} diff --git a/src/ContentFeed/DocumentBodyBuilder.php b/src/ContentFeed/DocumentBodyBuilder.php new file mode 100644 index 00000000..6913fe86 --- /dev/null +++ b/src/ContentFeed/DocumentBodyBuilder.php @@ -0,0 +1,288 @@ + [ + 'teaser' => null, + 'teaserLong' => 'Beschreibung', + 'feature' => 'Highlights', + 'conceptDescription' => 'Reisekonzept', + 'boardDescription' => 'Verpflegung', + 'includedServices' => 'Leistungen', + 'additionalServices' => 'Zusatzleistungen', + 'programmeDescription' => 'Programm', + 'skiPassDescription' => 'Skipass', + 'nonBookableInfo' => 'Hinweise zur Buchbarkeit', + ], + 'hotel' => [ + 'teaser' => null, + 'description' => 'Beschreibung', + 'features' => 'Ausstattung', + 'roomTypes' => 'Zimmer', + 'additionalInformation' => 'Weitere Informationen', + ], + 'region' => [ + 'teaser' => null, + 'description' => 'Beschreibung', + 'feature' => 'Highlights', + 'skiArea' => 'Skigebiet', + 'skiAreaExtended' => 'Skigebiet im Detail', + 'skiPass' => 'Skipass', + 'news' => 'Neuigkeiten', + ], + 'country' => [ + 'teaser' => null, + 'description' => 'Beschreibung', + 'descriptionExtended' => 'Mehr erfahren', + ], + 'city' => [ + 'teaser' => null, + 'description' => 'Beschreibung', + 'descriptionExtended' => 'Mehr erfahren', + 'party' => 'Party', + 'leisure' => 'Freizeit', + ], + 'concept' => [ + 'teaser' => null, + 'description' => 'Beschreibung', + 'descriptionExtended' => 'Mehr erfahren', + 'popupText' => 'Kurz erklärt', + ], + ]; + + /** + * Facts per type, with their German label. + */ + private const FACTS = [ + 'hotel' => [ + 'bedsCount' => 'Betten', + 'parkingCount' => 'Parkplätze', + 'altitude' => 'Höhenlage', + 'distanceBus' => 'Entfernung zur Bushaltestelle', + 'distanceLift' => 'Entfernung zum Lift', + 'shower' => 'Dusche/WC', + 'partyroom' => 'Partyraum', + 'gamesroom' => 'Spieleraum', + 'sauna' => 'Sauna', + 'wifi' => 'WLAN', + ], + 'region' => [ + 'length' => 'Pistenkilometer', + 'height' => 'Höhe', + 'lifts' => 'Lifte', + ], + ]; + + public function __construct(private readonly HtmlToMarkdownConverter $converter) + { + } + + /** + * @param array $record + */ + public function title(string $type, array $record): string + { + $title = 'news' === $type ? ($record['title'] ?? null) : ($record['name'] ?? null); + + return is_string($title) ? trim($title) : ''; + } + + /** + * @param array $record + */ + public function build(string $type, array $record): string + { + $baseUrl = is_string($record['url'] ?? null) ? $record['url'] : null; + $blocks = []; + + $title = $this->title($type, $record); + if ('' !== $title) { + $blocks[] = '# '.$title; + } + + if ('news' === $type) { + return $this->join(array_merge($blocks, $this->newsBlocks($record, $baseUrl))); + } + + foreach (self::LEAD_FIELDS as $field) { + $value = $this->plain($record[$field] ?? null); + if ('' !== $value && $value !== $title) { + $blocks[] = $value; + } + } + + if ('hotel' === $type) { + $address = $this->plain($record['address'] ?? null); + if ('' !== $address) { + $blocks[] = 'Adresse: '.$address; + } + } + + $text = is_array($record['text'] ?? null) ? $record['text'] : []; + foreach (self::TEXT_SECTIONS[$type] ?? [] as $field => $heading) { + $markdown = $this->converter->convert(is_string($text[$field] ?? null) ? $text[$field] : null, $baseUrl); + if ('' === $markdown) { + continue; + } + $blocks[] = null === $heading ? $markdown : "## {$heading}\n\n{$markdown}"; + } + + $facts = $this->facts($type, is_array($record['facts'] ?? null) ? $record['facts'] : []); + if ('' !== $facts) { + $blocks[] = "## Fakten\n\n".$facts; + } + + if ('region' === $type && is_string($record['webcam'] ?? null) && '' !== trim($record['webcam'])) { + $blocks[] = 'Webcam: '.trim($record['webcam']); + } + + if ('product' === $type) { + $faqs = $this->faqs(is_array($record['faqs'] ?? null) ? $record['faqs'] : [], $baseUrl); + if ('' !== $faqs) { + $blocks[] = "## Häufige Fragen\n\n".$faqs; + } + } + + return $this->join($blocks); + } + + /** + * @param array $record + * + * @return list + */ + private function newsBlocks(array $record, ?string $baseUrl): array + { + $blocks = []; + + if (is_string($record['datetime'] ?? null) && false !== ($date = date_create_immutable($record['datetime']))) { + $blocks[] = 'Veröffentlicht am '.$date->format('d.m.Y'); + } + + foreach (['teaser', 'bodytext'] as $field) { + $markdown = $this->converter->convert(is_string($record[$field] ?? null) ? $record[$field] : null, $baseUrl); + if ('' !== $markdown) { + $blocks[] = $markdown; + } + } + + $tags = $this->labels($record['tags'] ?? null); + if ([] !== $tags) { + $blocks[] = 'Themen: '.implode(', ', $tags); + } + + return $blocks; + } + + /** + * @param array $facts + */ + private function facts(string $type, array $facts): string + { + $lines = []; + + foreach (self::FACTS[$type] ?? [] as $field => $label) { + $value = $facts[$field] ?? null; + + $rendered = match (true) { + is_bool($value) => $value ? 'ja' : 'nein', + is_int($value), is_float($value) => (string) $value, + is_string($value) => $this->plain($value), + default => '', + }; + + if ('' !== $rendered) { + $lines[] = "- {$label}: {$rendered}"; + } + } + + return implode("\n", $lines); + } + + /** + * @param array $faqs + */ + private function faqs(array $faqs, ?string $baseUrl): string + { + $blocks = []; + + foreach ($faqs as $faq) { + if (!is_array($faq)) { + continue; + } + + $question = $this->plain($faq['question'] ?? null); + $answer = $this->converter->convert(is_string($faq['answer'] ?? null) ? $faq['answer'] : null, $baseUrl); + + if ('' !== $question && '' !== $answer) { + $blocks[] = "### {$question}\n\n{$answer}"; + } + } + + return $this->join($blocks); + } + + /** + * Tags and categories may arrive as strings or as objects with a title or name. + * + * @return list + */ + private function labels(mixed $values): array + { + if (!is_array($values)) { + return []; + } + + $labels = []; + foreach ($values as $value) { + $label = is_array($value) ? ($value['title'] ?? $value['name'] ?? null) : $value; + if (is_string($label) && '' !== trim($label)) { + $labels[] = trim($label); + } + } + + return $labels; + } + + /** + * A field that is plain text in the feed, defensively stripped of any markup. + */ + private function plain(mixed $value): string + { + if (!is_string($value)) { + return ''; + } + + return trim((string) preg_replace('/\s+/u', ' ', html_entity_decode(strip_tags($value), \ENT_QUOTES | \ENT_HTML5, 'UTF-8'))); + } + + /** + * @param list $blocks + */ + private function join(array $blocks): string + { + return implode("\n\n", $blocks); + } +} diff --git a/src/ContentFeed/Exception/ContentFeedStoppedException.php b/src/ContentFeed/Exception/ContentFeedStoppedException.php new file mode 100644 index 00000000..ba2513b6 --- /dev/null +++ b/src/ContentFeed/Exception/ContentFeedStoppedException.php @@ -0,0 +1,15 @@ +format('c') : '')); + } +} diff --git a/src/ContentFeed/Exception/ContentRequestFailedException.php b/src/ContentFeed/Exception/ContentRequestFailedException.php new file mode 100644 index 00000000..0e1a6450 --- /dev/null +++ b/src/ContentFeed/Exception/ContentRequestFailedException.php @@ -0,0 +1,13 @@ +converter = new HtmlConverter([ + 'header_style' => 'atx', + 'strip_tags' => true, + 'strip_placeholder_links' => true, + 'hard_break' => true, + 'suppress_errors' => true, + 'remove_nodes' => implode(' ', self::REMOVED_ELEMENTS), + ]); + $this->converter->getEnvironment()->addConverter(new TableConverter()); + } + + /** + * @param string|null $baseUrl what relative links resolve against; left as they are without one + */ + public function convert(?string $html, ?string $baseUrl = null): string + { + if (null === $html || '' === trim($html)) { + return ''; + } + + $markdown = $this->converter->convert($this->prepare($html, $baseUrl)); + + return $this->collapseWhitespace($markdown); + } + + private function prepare(string $html, ?string $baseUrl): string + { + $document = \Dom\HTMLDocument::createFromString(''.$html.'', \LIBXML_NOERROR, 'UTF-8'); + $body = $document->body; + + if (null === $body) { + return $html; + } + + foreach ($body->querySelectorAll(implode(',', self::REMOVED_ELEMENTS)) as $element) { + $element->remove(); + } + + // Whitespace in text collapses as a browser would; the template's indentation would + // otherwise end up inside link labels and headings. + foreach ((new \Dom\XPath($document))->query('//body//text()[not(ancestor::pre)]') as $text) { + if (!$text instanceof \Dom\Text) { + continue; + } + $text->data = (string) preg_replace('/[ \t\r\n]+/', ' ', $text->data); + } + + if (null !== $baseUrl) { + foreach ($body->querySelectorAll('a[href]') as $link) { + $href = trim((string) $link->getAttribute('href')); + + if ('' === $href || preg_match('/^(mailto|tel|javascript):/i', $href)) { + continue; + } + + $link->setAttribute('href', UriResolver::resolve($href, $baseUrl)); + } + } + + return $body->innerHTML; + } + + private function collapseWhitespace(string $markdown): string + { + // The converter leaves text entity-encoded (`&`), which only costs tokens. + $markdown = html_entity_decode($markdown, \ENT_QUOTES | \ENT_HTML5, 'UTF-8'); + $markdown = str_replace(["\r\n", "\r", "\u{00A0}"], ["\n", "\n", ' '], $markdown); + + // Labels padded by the collapsed whitespace: "[ Kontakt ](…)" -> "[Kontakt](…)". + $markdown = (string) preg_replace(['/\[ +/', '/ +\]\(/'], ['[', ']('], $markdown); + + $lines = array_map( + // Inner runs only: leading spaces carry list nesting. + static fn (string $line): string => rtrim((string) preg_replace('/(?<=\S)[ \t]{2,}/', ' ', $line)), + explode("\n", $markdown), + ); + + return trim((string) preg_replace("/\n{3,}/", "\n\n", implode("\n", $lines))); + } +} diff --git a/src/Entity/ContentDocument.php b/src/Entity/ContentDocument.php new file mode 100644 index 00000000..165bfcb0 --- /dev/null +++ b/src/Entity/ContentDocument.php @@ -0,0 +1,325 @@ +|null + */ + #[ORM\Column(type: Types::JSON, nullable: true)] + private ?array $data = null; + + /** + * The manifest's `modified`, as last seen. + */ + #[ORM\Column(type: Types::DATETIME_IMMUTABLE)] + private \DateTimeImmutable $modified; + + /** + * The manifest's `modified` that the stored fetch outcome belongs to. The item is stale + * while this differs from `modified`. + */ + #[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)] + private ?\DateTimeImmutable $fetchedModified = null; + + #[ORM\Column(length: 16, enumType: ContentFetchState::class)] + private ContentFetchState $fetchState = ContentFetchState::Pending; + + /** + * Last fetch attempt that produced an outcome, whichever it was. + */ + #[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)] + private ?\DateTimeImmutable $syncedAt = null; + + /** + * `type:uid` of the record this page is the detail page of, in the same language. + */ + #[ORM\Column(length: 32, nullable: true)] + private ?string $mergedInto = null; + + #[ORM\Column] + private bool $indexed = false; + + public function __construct(string $type, int $uid, string $language, \DateTimeImmutable $modified) + { + $this->type = $type; + $this->uid = $uid; + $this->language = $language; + $this->modified = $modified; + } + + public static function key(string $type, int $uid, string $language): string + { + return $type.':'.$uid.':'.$language; + } + + public function getKey(): string + { + return self::key($this->type, $this->uid, $this->language); + } + + /** + * The `type:uid` a merged page refers to its record by. + */ + public function getRecordRef(): string + { + return $this->type.':'.$this->uid; + } + + public function isPage(): bool + { + return self::TYPE_PAGE === $this->type; + } + + public function isStale(): bool + { + return null === $this->fetchedModified || $this->fetchedModified != $this->modified; + } + + public function getId(): ?int + { + return $this->id; + } + + public function getType(): string + { + return $this->type; + } + + public function getUid(): int + { + return $this->uid; + } + + public function getLanguage(): string + { + return $this->language; + } + + public function getSite(): ?string + { + return $this->site; + } + + public function setSite(?string $site): self + { + $this->site = $site; + + return $this; + } + + public function getUrl(): ?string + { + return $this->url; + } + + public function setUrl(?string $url): self + { + $this->url = $url; + + return $this; + } + + public function getTitle(): string + { + return $this->title; + } + + public function setTitle(string $title): self + { + $this->title = mb_substr($title, 0, 255); + + return $this; + } + + public function getBreadcrumb(): ?string + { + return $this->breadcrumb; + } + + public function setBreadcrumb(?string $breadcrumb): self + { + $this->breadcrumb = null === $breadcrumb ? null : mb_substr($breadcrumb, 0, 512); + + return $this; + } + + public function getBodyMd(): string + { + return $this->bodyMd; + } + + public function setBodyMd(string $bodyMd): self + { + $this->bodyMd = $bodyMd; + $this->updateIndexed(); + + return $this; + } + + /** + * @return array|null + */ + public function getData(): ?array + { + return $this->data; + } + + /** + * @param array|null $data + */ + public function setData(?array $data): self + { + $this->data = $data; + $this->updateIndexed(); + + return $this; + } + + public function getModified(): \DateTimeImmutable + { + return $this->modified; + } + + public function setModified(\DateTimeImmutable $modified): self + { + // Compared by value: assigning an equal instance would still schedule an update. + if ($modified != $this->modified) { + $this->modified = $modified; + } + + return $this; + } + + public function getFetchedModified(): ?\DateTimeImmutable + { + return $this->fetchedModified; + } + + public function getFetchState(): ContentFetchState + { + return $this->fetchState; + } + + public function getSyncedAt(): ?\DateTimeImmutable + { + return $this->syncedAt; + } + + /** + * Records the outcome of a fetch against the `modified` it was made for. + */ + public function markFetched(ContentFetchState $state, \DateTimeImmutable $at): self + { + $this->fetchState = $state; + $this->fetchedModified = $this->modified; + $this->syncedAt = $at; + $this->updateIndexed(); + + return $this; + } + + public function getMergedInto(): ?string + { + return $this->mergedInto; + } + + public function setMergedInto(?string $mergedInto): self + { + $this->mergedInto = $mergedInto; + $this->updateIndexed(); + + return $this; + } + + public function isIndexed(): bool + { + return $this->indexed; + } + + private function updateIndexed(): void + { + $this->indexed = ContentFetchState::Ok === $this->fetchState + && '' !== $this->bodyMd + && null === $this->mergedInto + && !$this->isHiddenFromSearch(); + } + + /** + * TYPO3's own "not in the website search" flag, on products. + */ + public function isHiddenFromSearch(): bool + { + return 'product' === $this->type && false === ($this->data['searchable'] ?? null); + } +} diff --git a/src/Entity/ContentSyncState.php b/src/Entity/ContentSyncState.php new file mode 100644 index 00000000..7bea6cd4 --- /dev/null +++ b/src/Entity/ContentSyncState.php @@ -0,0 +1,97 @@ +id; + } + + public function getManifestGeneratedAt(): ?\DateTimeImmutable + { + return $this->manifestGeneratedAt; + } + + public function setManifestGeneratedAt(?\DateTimeImmutable $manifestGeneratedAt): self + { + $this->manifestGeneratedAt = $manifestGeneratedAt; + + return $this; + } + + public function getLastRunAt(): ?\DateTimeImmutable + { + return $this->lastRunAt; + } + + public function setLastRunAt(?\DateTimeImmutable $lastRunAt): self + { + $this->lastRunAt = $lastRunAt; + + return $this; + } + + public function getLastError(): ?string + { + return $this->lastError; + } + + public function setLastError(?string $lastError): self + { + $this->lastError = $lastError; + + return $this; + } + + public function getPausedUntil(): ?\DateTimeImmutable + { + return $this->pausedUntil; + } + + public function setPausedUntil(?\DateTimeImmutable $pausedUntil): self + { + $this->pausedUntil = $pausedUntil; + + return $this; + } +} diff --git a/src/Enum/ContentFetchState.php b/src/Enum/ContentFetchState.php new file mode 100644 index 00000000..cb0b03f7 --- /dev/null +++ b/src/Enum/ContentFetchState.php @@ -0,0 +1,23 @@ + $items keyed by ManifestItem::key() + * @param list $siteHosts hosts of the sites' base URLs, the only ones a contentUrl may point to + */ + public function __construct( + public \DateTimeImmutable $generatedAt, + public array $items, + public array $siteHosts, + ) { + } +} diff --git a/src/Model/ContentFeed/ContentSyncReport.php b/src/Model/ContentFeed/ContentSyncReport.php new file mode 100644 index 00000000..51bb4ad3 --- /dev/null +++ b/src/Model/ContentFeed/ContentSyncReport.php @@ -0,0 +1,42 @@ +stoppedBy && null === $this->pagePhaseEndedBy && 0 === $this->deferred; + } + + public function changedAnything(): bool + { + return $this->added + $this->deleted + $this->records + $this->pages > 0; + } + + /** + * @return array + */ + public function toLogContext(): array + { + return get_object_vars($this); + } +} diff --git a/src/Model/ContentFeed/ManifestItem.php b/src/Model/ContentFeed/ManifestItem.php new file mode 100644 index 00000000..363bd33b --- /dev/null +++ b/src/Model/ContentFeed/ManifestItem.php @@ -0,0 +1,90 @@ + $breadcrumb + * @param list $detailOf + */ + public function __construct( + public string $type, + public int $uid, + public string $language, + public ?string $site, + public ?string $url, + public \DateTimeImmutable $modified, + public ?string $contentUrl = null, + public ?string $title = null, + public ?string $description = null, + public array $breadcrumb = [], + public array $detailOf = [], + ) { + } + + /** + * @param array $item + */ + public static function fromArray(array $item): self + { + if (!is_string($item['type'] ?? null) || !is_int($item['uid'] ?? null) || !is_string($item['language'] ?? null) || !is_string($item['modified'] ?? null)) { + throw new \InvalidArgumentException('Manifest item without type, uid, language or modified'); + } + + $modified = self::localTime($item['modified']); + if (null === $modified) { + throw new \InvalidArgumentException(sprintf('Manifest item %s:%d has an unparseable modified', $item['type'], $item['uid'])); + } + + $detailOf = []; + foreach (is_array($item['detailOf'] ?? null) ? $item['detailOf'] : [] as $ref) { + if (is_array($ref) && is_string($ref['type'] ?? null) && is_int($ref['uid'] ?? null)) { + $detailOf[] = ['type' => $ref['type'], 'uid' => $ref['uid']]; + } + } + + $string = static fn (mixed $value): ?string => is_string($value) && '' !== $value ? $value : null; + + return new self( + type: $item['type'], + uid: $item['uid'], + language: $item['language'], + site: $string($item['site'] ?? null), + url: $string($item['url'] ?? null), + modified: $modified, + contentUrl: $string($item['contentUrl'] ?? null), + title: $string($item['title'] ?? null), + description: $string($item['description'] ?? null), + breadcrumb: array_values(array_filter(is_array($item['breadcrumb'] ?? null) ? $item['breadcrumb'] : [], 'is_string')), + detailOf: $detailOf, + ); + } + + /** + * A feed timestamp in the application's timezone. Doctrine stores datetimes without + * their offset, so a value kept in TYPO3's +02:00 would never compare equal to itself + * once reloaded, and every item would look changed on every run. + */ + public static function localTime(string $value): ?\DateTimeImmutable + { + $date = date_create_immutable($value); + + return false === $date ? null : $date->setTimezone(new \DateTimeZone(date_default_timezone_get())); + } + + public function key(): string + { + return ContentDocument::key($this->type, $this->uid, $this->language); + } +} diff --git a/src/Model/ContentFeed/PageFetchResult.php b/src/Model/ContentFeed/PageFetchResult.php new file mode 100644 index 00000000..5d85c257 --- /dev/null +++ b/src/Model/ContentFeed/PageFetchResult.php @@ -0,0 +1,41 @@ +html; + } +} diff --git a/src/Repository/ContentDocumentRepository.php b/src/Repository/ContentDocumentRepository.php new file mode 100644 index 00000000..2d4dea80 --- /dev/null +++ b/src/Repository/ContentDocumentRepository.php @@ -0,0 +1,166 @@ + + */ +class ContentDocumentRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, ContentDocument::class); + } + + /** + * @return array keyed by ContentDocument::getKey() + */ + public function findAllByKey(): array + { + $documents = []; + + foreach ($this->findAll() as $document) { + $documents[$document->getKey()] = $document; + } + + return $documents; + } + + /** + * Full-text search over the indexed documents that can be linked to. + * + * DBAL rather than DQL: Doctrine has no MATCH ... AGAINST. The score is weighted by type + * so that, at similar relevance, what people book ranks above what they read. + * + * @param string $booleanQuery a MySQL boolean-mode query, built by ContentSearchQuery + * @param list $types empty for all + * @param array $typeWeights + * + * @return list + */ + public function searchFulltext(string $booleanQuery, array $types, ?string $site, array $typeWeights, int $limit): array + { + $weight = 'CASE type'; + $params = ['query' => $booleanQuery, 'limit' => $limit]; + $paramTypes = ['limit' => ParameterType::INTEGER]; + + foreach (array_keys($typeWeights) as $i => $type) { + $weight .= sprintf(' WHEN :wt%d THEN :wv%d', $i, $i); + $params['wt'.$i] = $type; + $params['wv'.$i] = $typeWeights[$type]; + } + $weight .= ' ELSE 1 END'; + + $sql = 'SELECT type, uid, title, url, site, breadcrumb, body_md,' + .' MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE) * '.$weight.' AS score' + .' FROM content_document' + .' WHERE indexed = 1 AND url IS NOT NULL' + .' AND MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE)'; + + if ([] !== $types) { + $sql .= ' AND type IN (:types)'; + $params['types'] = $types; + $paramTypes['types'] = ArrayParameterType::STRING; + } + + if (null !== $site) { + $sql .= ' AND site = :site'; + $params['site'] = $site; + } + + $sql .= ' ORDER BY score DESC, uid ASC LIMIT :limit'; + + $rows = $this->getEntityManager()->getConnection()->fetchAllAssociative($sql, $params, $paramTypes); + + return array_map(static fn (array $row): array => [ + 'type' => (string) $row['type'], + 'uid' => (int) $row['uid'], + 'title' => (string) $row['title'], + 'url' => (string) $row['url'], + 'site' => null === $row['site'] ? null : (string) $row['site'], + 'breadcrumb' => null === $row['breadcrumb'] ? null : (string) $row['breadcrumb'], + 'body_md' => (string) $row['body_md'], + 'score' => (float) $row['score'], + ], $rows); + } + + /** + * The `(type, uid)` of every fetched record whose text matches, for resolving free text + * in the trip search. Unlike searchFulltext(), records without a URL count: a hotel with + * no page of its own still leads to the products it belongs to. + * + * @param list $types + * + * @return list + */ + public function matchRecords(string $booleanQuery, array $types, string $language): array + { + $rows = $this->getEntityManager()->getConnection()->fetchAllAssociative( + 'SELECT type, uid FROM content_document' + .' WHERE fetch_state = :ok AND language = :language AND type IN (:types)' + .' AND MATCH(title, breadcrumb, body_md) AGAINST(:query IN BOOLEAN MODE)', + ['ok' => ContentFetchState::Ok->value, 'language' => $language, 'types' => $types, 'query' => $booleanQuery], + ['types' => ArrayParameterType::STRING], + ); + + return array_map(static fn (array $row): array => ['type' => (string) $row['type'], 'uid' => (int) $row['uid']], $rows); + } + + /** + * Fetched records of the given types in one language, keyed by `type:uid`. + * + * @param list $types + * + * @return array + */ + public function findRecords(array $types, string $language): array + { + /** @var list $documents */ + $documents = $this->createQueryBuilder('d') + ->where('d.type IN (:types)') + ->andWhere('d.language = :language') + ->andWhere('d.fetchState = :ok') + ->setParameter('types', $types) + ->setParameter('language', $language) + ->setParameter('ok', ContentFetchState::Ok) + ->getQuery() + ->getResult(); + + $indexed = []; + + foreach ($documents as $document) { + $indexed[$document->getRecordRef()] = $document; + } + + return $indexed; + } + + /** + * The indexed document published at a URL. Trailing slashes are not significant. + */ + public function findOneIndexedByUrl(string $url): ?ContentDocument + { + $trimmed = rtrim($url, '/'); + + /** @var list $documents */ + $documents = $this->createQueryBuilder('d') + ->where('d.url IN (:urls)') + ->andWhere('d.indexed = true') + ->setParameter('urls', [$trimmed, $trimmed.'/']) + ->orderBy('d.id', 'ASC') + ->setMaxResults(1) + ->getQuery() + ->getResult(); + + return $documents[0] ?? null; + } +} diff --git a/src/Repository/ContentSyncStateRepository.php b/src/Repository/ContentSyncStateRepository.php new file mode 100644 index 00000000..767a30ef --- /dev/null +++ b/src/Repository/ContentSyncStateRepository.php @@ -0,0 +1,28 @@ + + */ +class ContentSyncStateRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, ContentSyncState::class); + } + + /** + * The single state row, created unpersisted on first use. + */ + public function get(): ContentSyncState + { + return $this->findOneBy([], ['id' => 'ASC']) ?? new ContentSyncState(); + } +} diff --git a/src/Security/McpAccessTokenHandler.php b/src/Security/McpAccessTokenHandler.php new file mode 100644 index 00000000..24e3c588 --- /dev/null +++ b/src/Security/McpAccessTokenHandler.php @@ -0,0 +1,37 @@ +token || !hash_equals($this->token, $accessToken)) { + throw new BadCredentialsException('Invalid MCP token.'); + } + + return new UserBadge(self::USER); + } +} diff --git a/symfony.lock b/symfony.lock index a74134c4..1a947e53 100644 --- a/symfony.lock +++ b/symfony.lock @@ -103,6 +103,18 @@ "config/packages/nyholm_psr7.yaml" ] }, + "php-http/discovery": { + "version": "1.20", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "1.18", + "ref": "f45b5dd173a27873ab19f5e3180b2f661c21de02" + }, + "files": [ + "config/packages/http_discovery.yaml" + ] + }, "phpstan/phpstan": { "version": "2.1", "recipe": { @@ -241,6 +253,9 @@ "ref": "fadbfe33303a76e25cb63401050439aa9b1a9c7f" } }, + "symfony/mcp-bundle": { + "version": "v0.14.1" + }, "symfony/messenger": { "version": "6.4", "recipe": { diff --git a/tests/Chatbot/Search/ContentSearchQueryTest.php b/tests/Chatbot/Search/ContentSearchQueryTest.php new file mode 100644 index 00000000..088944dd --- /dev/null +++ b/tests/Chatbot/Search/ContentSearchQueryTest.php @@ -0,0 +1,47 @@ +catalog([ + $this->row('VTH', 1, '2026-09-01', 'HOT1', [$this->room(100)]), + $this->row('VTH', 2, '2026-12-05', 'HOT1', [$this->room(500), $this->room(450, status: 'Anfrage'), $this->room(99, status: 'Buchungsstop')]), + $this->row('VTH', 3, '2027-01-09', 'HOT1', [$this->room(520, status: 'Anfrage')]), + ])->search(new TripCriteria()); + + self::assertCount(1, $result['results']); + $trip = $result['results'][0]; + self::assertSame('VTH', $trip['code']); + self::assertSame('Val Thorens Apartments', $trip['name']); + self::assertSame('https://www.ep-reisen.de/val-thorens/', $trip['url']); + self::assertSame(['country' => 'Frankreich'], $trip['destination']); + self::assertSame(['Hotel Eins'], $trip['hotels']); + self::assertSame([ + ['dateFrom' => '2026-12-05', 'dateTo' => '2026-12-12', 'priceFrom' => 450.0, 'availability' => 'frei'], + ['dateFrom' => '2027-01-09', 'dateTo' => '2027-01-16', 'priceFrom' => 520.0, 'availability' => 'auf Anfrage'], + ], $trip['nextDates']); + self::assertSame('2026-10-08T06:00:00+00:00', $result['dataAsOf']); + } + + public function testPersonsCountsPlacesAcrossBookableRoomsOnly(): void + { + $rows = [ + // 2 rooms x 2 pax + 1 x 1 pax = 5 places; the stopped room's 10 do not count. + $this->row('VTH', 2, '2026-12-05', 'HOT1', [$this->room(400, available: 2, maxPax: 2), $this->room(300, available: 1, maxPax: 1), $this->room(200, available: 10, maxPax: 1, status: 'Buchungsstop')]), + ]; + + self::assertCount(1, $this->catalog($rows)->search(new TripCriteria(persons: 5))['results']); + self::assertCount(0, $this->catalog($rows)->search(new TripCriteria(persons: 6))['results']); + } + + public function testDatePeriodMatchesOnOverlapAndMaxPriceOnTheLowestPrice(): void + { + $rows = [$this->row('VTH', 2, '2026-12-05', 'HOT1', [$this->room(450), $this->room(700)])]; + + self::assertCount(1, $this->catalog($rows)->search(new TripCriteria(dateFrom: new \DateTimeImmutable('2026-12-12'), dateTo: new \DateTimeImmutable('2026-12-31')))['results']); + self::assertCount(0, $this->catalog($rows)->search(new TripCriteria(dateFrom: new \DateTimeImmutable('2026-12-13')))['results']); + self::assertCount(1, $this->catalog($rows)->search(new TripCriteria(maxPrice: 450))['results']); + self::assertCount(0, $this->catalog($rows)->search(new TripCriteria(maxPrice: 449.99))['results']); + } + + public function testCountryAndConceptAreResolvedThroughTheContentRecords(): void + { + $rows = [$this->row('VTH', 2, '2026-12-05', 'HOT1', [$this->room(450)])]; + + self::assertCount(1, $this->catalog($rows)->search(new TripCriteria(country: 'frankreich', concept: 'ski'))['results']); + self::assertCount(0, $this->catalog($rows)->search(new TripCriteria(country: 'Österreich'))['results']); + self::assertCount(0, $this->catalog($rows)->search(new TripCriteria(concept: 'party'))['results']); + } + + public function testGetJoinsHotelsOnTheirAliasesAndPreferTheOwner(): void + { + $trip = $this->catalog([ + $this->row('VTH', 2, '2026-12-05', 'HOT1ALIAS', [$this->room(450)]), + $this->row('VTH', 2, '2026-12-05', 'HOT2', [$this->room(380, status: 'Buchungsstop')]), + ])->get('vth', 'HOT1'); + + self::assertNotNull($trip); + self::assertSame(['Hotel Eins'], array_column($trip['hotels'], 'name')); + self::assertCount(1, $trip['dates']); + self::assertSame('Hotel Eins', $trip['dates'][0]['hotels'][0]['hotel']); + self::assertSame([['room' => 'Zimmer', 'board' => 'HP', 'nights' => 7, 'price' => 450, 'availability' => 'frei']], $trip['dates'][0]['hotels'][0]['rooms']); + self::assertSame([['city' => 'Frankfurt', 'place' => 'Flughafen', 'departure' => '2026-12-04 23:30', 'surcharge' => 5.9]], $trip['dates'][0]['pickups']); + self::assertArrayNotHasKey('guide', $trip['dates'][0]); + } + + public function testProductsHiddenFromTheWebsiteSearchAreNotOfferedButCanBeNamed(): void + { + $rows = [$this->row('GRP', 2, '2026-12-05', 'HOT1', [$this->room(300)])]; + + self::assertSame([], $this->catalog($rows)->search(new TripCriteria())['results']); + self::assertSame('GRP', $this->catalog($rows)->get('GRP')['code'] ?? null); + } + + public function testGetIsNullForACodeWithoutAPublishedProduct(): void + { + self::assertNull($this->catalog([$this->row('XYZ', 2, '2026-12-05', 'HOT1', [$this->room(1)])])->get('XYZ')); + } + + /** + * @param list> $rows + */ + private function catalog(array $rows): TripCatalog + { + $export = $this->createStub(ProductExportBuilder::class); + $export->method('build')->willReturn(new Export($rows, 0, 0, new \DateTimeImmutable('2026-10-08 09:00:00+00:00'), new \DateTimeImmutable('2026-10-08 06:00:00+00:00'))); + + $records = []; + foreach ([ + $this->record('product', 1, 'Val Thorens Apartments', 'https://www.ep-reisen.de/val-thorens/', ['code' => 'VTH', 'countryUid' => 2, 'conceptUid' => 7, 'hotelUids' => [10], 'daytrip' => false, 'bookable' => true]), + $this->record('product', 2, 'Gruppenreise Uni X', 'https://www.ep-reisen.de/gruppe/', ['code' => 'GRP', 'countryUid' => 2, 'searchable' => false, 'bookable' => true]), + $this->record('hotel', 10, 'Hotel Eins', null, ['code' => 'HOT1', 'codes' => ['HOT1', 'HOT1ALIAS', 'SHARED'], 'facts' => ['wifi' => true, 'sauna' => null]]), + $this->record('hotel', 11, 'Hotel Zwei', null, ['code' => 'SHARED', 'codes' => ['HOT2']]), + $this->record('country', 2, 'Frankreich', null, ['code' => 'FR']), + $this->record('concept', 7, 'Skiurlaub ohne Programm', null, ['code' => 'ski']), + ] as $record) { + $records[$record->getRecordRef()] = $record; + } + + $repository = $this->createStub(ContentDocumentRepository::class); + $repository->method('findRecords')->willReturn($records); + $repository->method('matchRecords')->willReturn([]); + + return new TripCatalog($export, $repository, new MockClock('2026-10-08 10:00:00'), new NullLogger()); + } + + /** + * @param array $data + */ + private function record(string $type, int $uid, string $title, ?string $url, array $data): ContentDocument + { + return (new ContentDocument($type, $uid, 'de', new \DateTimeImmutable())) + ->setTitle($title) + ->setUrl($url) + ->setSite(null === $url ? null : 'ep-reisen') + ->setData($data + ['uid' => $uid]) + ->setBodyMd('# '.$title) + ->markFetched(ContentFetchState::Ok, new \DateTimeImmutable()); + } + + /** + * @param list> $rooms + * + * @return array + */ + private function row(string $code, int $dateId, string $from, string $hotelCode, array $rooms): array + { + $dateFrom = new \DateTimeImmutable($from); + + return [ + 'product' => ['code' => $code, 'busProId' => 1], + 'date' => ['code' => $code.$dateId, 'busProId' => $dateId, 'from' => $from, 'to' => $dateFrom->modify('+7 days')->format('Y-m-d'), 'type' => 'F'], + 'hotel' => ['code' => $hotelCode, 'busProId' => crc32($hotelCode)], + 'rooms' => $rooms, + 'pickups' => [ + ['busProId' => 4, 'code' => 'FRA', 'city' => 'Frankfurt', 'street' => 'Flughafen', 'price' => 5.9, 'datetime' => $dateFrom->modify('-1 day')->format('Y-m-d').'T23:30:00'], + ['busProId' => 0, 'code' => null, 'city' => null, 'street' => null, 'price' => null, 'datetime' => null], + ], + 'guide' => ['name' => 'Anna', 'phone' => '0170'], + ]; + } + + /** + * @return array + */ + private function room(int $price, int $available = 5, int $maxPax = 2, string $status = 'Frei'): array + { + return ['code' => 'DZ', 'label' => 'Zimmer', 'minPax' => 1, 'maxPax' => $maxPax, 'nights' => 7, 'price' => $price, 'available' => $available, 'status' => $status, 'boardCode' => 'HP']; + } +} diff --git a/tests/ContentFeed/ContentFeedClientTest.php b/tests/ContentFeed/ContentFeedClientTest.php new file mode 100644 index 00000000..e3e1873a --- /dev/null +++ b/tests/ContentFeed/ContentFeedClientTest.php @@ -0,0 +1,257 @@ +clock = new MockClock('2026-10-08 10:00:00'); + } + + public function testManifestIsParsedAndSendsTheApiKey(): void + { + $requests = []; + $client = $this->client([ + function (string $method, string $url, array $options) use (&$requests): MockResponse { + $requests[] = [$url, $options['normalized_headers']['x-api-key'][0] ?? null]; + + return $this->manifestResponse(); + }, + ]); + + $manifest = $client->manifest(); + + self::assertSame([['https://www.ep-reisen.de/api/content/manifest', 'X-Api-Key: secret']], $requests); + self::assertCount(2, $manifest->items); + self::assertSame(['www.ep-reisen.de', 'snowzone.net'], $manifest->siteHosts); + self::assertSame('2026-10-08 08:15:00 UTC', $manifest->generatedAt->setTimezone(new \DateTimeZone('UTC'))->format('Y-m-d H:i:s T')); + } + + /** + * @return iterable + */ + public static function unusableManifests(): iterable + { + yield 'server error' => [new MockResponse('', ['http_code' => 500])]; + yield 'forbidden' => [new MockResponse('', ['http_code' => 403])]; + yield 'invalid json' => [new MockResponse('{"items":')]; + yield 'empty items' => [new MockResponse('{"generatedAt":"2026-10-08T10:15:00+02:00","sites":[],"items":[]}')]; + yield 'malformed item' => [new MockResponse('{"generatedAt":"2026-10-08T10:15:00+02:00","sites":[],"items":[{"type":"page"}]}')]; + } + + /** + * @dataProvider unusableManifests + */ + #[\PHPUnit\Framework\Attributes\DataProvider('unusableManifests')] + public function testAnUnusableManifestStopsTheRun(MockResponse $response): void + { + $this->expectException(ContentFeedStoppedException::class); + + $this->client([$response])->manifest(); + } + + public function testMissingApiKeyStopsWithoutARequest(): void + { + $this->expectException(ContentFeedStoppedException::class); + $this->expectExceptionMessage('APP_CMS_API_KEY'); + + (new ContentFeedClient(new MockHttpClient([]), $this->clock, new NullLogger(), self::BASE, ''))->manifest(); + } + + public function test503EndsTheRunWithRetryAfter(): void + { + try { + $this->client([new MockResponse('', ['http_code' => 503, 'response_headers' => ['Retry-After' => '3600']])])->manifest(); + self::fail('Expected the run to stop'); + } catch (ContentFeedUnavailableException $e) { + self::assertSame($this->clock->now()->getTimestamp() + 3600, $e->retryAfter?->getTimestamp()); + } + } + + public function testRecordsAreKeyedByUidFromTheEnvelope(): void + { + $client = $this->client([new MockResponse('{"type":"product","items":[{"uid":412,"name":"A"},{"uid":413,"name":"B"}]}')]); + + $records = $client->records('product', [412, 413, 414]); + + self::assertSame([412, 413], array_keys($records)); + } + + public function testARecordsResponseWithoutTheEnvelopeFailsTheBatchInsteadOfDeleting(): void + { + $this->expectException(ContentRequestFailedException::class); + + $this->client([new MockResponse('[{"uid":412}]')])->records('product', [412]); + } + + public function testThreeConsecutiveServerErrorsOrTimeoutsStopTheRun(): void + { + $client = $this->client([ + new MockResponse('', ['http_code' => 500]), + new MockResponse([new TransportException('timeout')]), + new MockResponse('', ['http_code' => 502]), + ]); + + foreach ([1, 2] as $attempt) { + try { + $client->records('product', [$attempt]); + self::fail('Expected the batch to fail'); + } catch (ContentRequestFailedException) { + } + } + + $this->expectException(ContentFeedStoppedException::class); + $client->records('product', [3]); + } + + public function testASuccessResetsTheFailureCount(): void + { + $client = $this->client([ + new MockResponse('', ['http_code' => 500]), + new MockResponse('', ['http_code' => 500]), + new MockResponse('{"type":"product","items":[]}'), + new MockResponse('', ['http_code' => 500]), + ]); + + $failures = 0; + foreach ([1, 2, 3, 4] as $uid) { + try { + $client->records('product', [$uid]); + } catch (ContentRequestFailedException) { + ++$failures; + } + } + + self::assertSame(3, $failures); + } + + public function testASinglePageErrorIsAFailureOfThatPageOnly(): void + { + $client = $this->clientWithManifest([ + new MockResponse('', ['http_code' => 404]), + new MockResponse('', ['http_code' => 500]), + new MockResponse('

A

'), + ]); + + self::assertSame(404, $client->page('https://www.ep-reisen.de/a?type=1730')->failedStatus); + self::assertSame(500, $client->page('https://www.ep-reisen.de/b?type=1730')->failedStatus); + self::assertSame('

A

', $client->page('https://www.ep-reisen.de/c?type=1730')->html); + } + + public function testAPageTimeoutIsTransient(): void + { + $client = $this->clientWithManifest([new MockResponse([new TransportException('timeout')])]); + + self::assertTrue($client->page('https://www.ep-reisen.de/a?type=1730')->transient); + } + + public function testPagesAreFetchedAtMostOncePerSecond(): void + { + $client = $this->clientWithManifest([new MockResponse('

a

'), new MockResponse('

b

'), new MockResponse('

c

')]); + $start = $this->clock->now(); + + $client->page('https://www.ep-reisen.de/a?type=1730'); + $client->page('https://www.ep-reisen.de/b?type=1730'); + $client->page('https://www.ep-reisen.de/c?type=1730'); + + self::assertSame(2, $this->clock->now()->getTimestamp() - $start->getTimestamp()); + } + + public function testASlowResponseDelaysTheNextRequestByFiveSeconds(): void + { + $client = $this->clientWithManifest([new MockResponse('

a

', ['total_time' => 2.5]), new MockResponse('

b

')]); + $start = $this->clock->now(); + + $client->page('https://www.ep-reisen.de/a?type=1730'); + $client->page('https://www.ep-reisen.de/b?type=1730'); + + self::assertSame(5, $this->clock->now()->getTimestamp() - $start->getTimestamp()); + } + + public function testThreeSlowPagesInARowEndThePagePhase(): void + { + $client = $this->clientWithManifest([ + new MockResponse('

a

', ['total_time' => 3.0]), + new MockResponse('

b

', ['total_time' => 3.0]), + new MockResponse('

c

', ['total_time' => 3.0]), + ]); + + $client->page('https://www.ep-reisen.de/a?type=1730'); + $client->page('https://www.ep-reisen.de/b?type=1730'); + + $this->expectException(PagePhaseEndedException::class); + $client->page('https://www.ep-reisen.de/c?type=1730'); + } + + public function testTheKeyIsNeverSentToAHostOutsideTheManifestSites(): void + { + $client = $this->clientWithManifest([]); + + $result = $client->page('https://evil.example/a?type=1730'); + + self::assertFalse($result->isOk()); + self::assertFalse($result->transient); + } + + /** + * @param list $responses + */ + private function client(array $responses): ContentFeedClient + { + $client = new ContentFeedClient(new MockHttpClient($responses, self::BASE), $this->clock, new NullLogger(), self::BASE, 'secret'); + $client->startRun(); + + return $client; + } + + /** + * A client that has read the manifest, so it knows the sites' hosts. + * + * @param list $responses + */ + private function clientWithManifest(array $responses): ContentFeedClient + { + $client = $this->client(array_merge([$this->manifestResponse()], $responses)); + $client->manifest(); + + return $client; + } + + private function manifestResponse(): MockResponse + { + return new MockResponse(json_encode([ + 'generatedAt' => '2026-10-08T10:15:00+02:00', + 'sites' => [ + ['identifier' => 'ep-reisen', 'base' => 'https://www.ep-reisen.de/'], + ['identifier' => 'snowzone', 'base' => 'https://snowzone.net/'], + ], + 'items' => [ + ['type' => 'page', 'uid' => 1, 'language' => 'de', 'site' => 'ep-reisen', 'url' => 'https://www.ep-reisen.de/a', 'contentUrl' => 'https://www.ep-reisen.de/a?type=1730', 'modified' => '2026-09-30T14:02:11+02:00'], + ['type' => 'product', 'uid' => 412, 'language' => 'de', 'site' => 'ep-reisen', 'url' => null, 'modified' => '2026-09-12T09:40:00+02:00'], + ], + ], \JSON_THROW_ON_ERROR)); + } +} diff --git a/tests/ContentFeed/ContentSyncerTest.php b/tests/ContentFeed/ContentSyncerTest.php new file mode 100644 index 00000000..1a5ca1ea --- /dev/null +++ b/tests/ContentFeed/ContentSyncerTest.php @@ -0,0 +1,227 @@ + what the fake database holds */ + private array $stored = []; + + private ContentSyncState $state; + + private ContentFeedClient&Stub $client; + + protected function setUp(): void + { + $this->state = new ContentSyncState(); + $this->client = $this->createStub(ContentFeedClient::class); + } + + public function testAFailedManifestChangesNothing(): void + { + $this->stored = ['page:1:de' => $this->document('page', 1)]; + $this->client->method('manifest')->willThrowException(new ContentFeedStoppedException('Manifest has no items')); + + $report = $this->syncer()->run(); + + self::assertSame('Manifest has no items', $report->stoppedBy); + self::assertArrayHasKey('page:1:de', $this->stored); + self::assertNull($this->state->getManifestGeneratedAt()); + self::assertSame('Manifest has no items', $this->state->getLastError()); + } + + public function testItemsMissingFromTheManifestAreDeleted(): void + { + $this->stored = ['page:1:de' => $this->document('page', 1), 'page:2:de' => $this->document('page', 2)]; + $this->client->method('manifest')->willReturn($this->manifest([$this->pageItem(1)])); + $this->client->method('page')->willReturn(PageFetchResult::ok('

Eins

')); + + $report = $this->syncer()->run(); + + self::assertSame(1, $report->deleted); + self::assertSame(['page:1:de'], array_keys($this->stored)); + self::assertEquals(new \DateTimeImmutable('2026-10-08 08:15:00'), $this->state->getManifestGeneratedAt()); + } + + public function testRecordsAbsentFromTheResponseAreDeleted(): void + { + $this->client->method('manifest')->willReturn($this->manifest([$this->recordItem('product', 412), $this->recordItem('product', 413)])); + $this->client->method('records')->willReturn([412 => ['uid' => 412, 'name' => 'Val Thorens', 'code' => 'VTH', 'text' => ['teaser' => '

Ski

']]]); + + $report = $this->syncer()->run(); + + self::assertSame(1, $report->records); + self::assertSame(['product:412:de'], array_keys($this->stored)); + self::assertSame("# Val Thorens\n\nSki", $this->stored['product:412:de']->getBodyMd()); + self::assertTrue($this->stored['product:412:de']->isIndexed()); + } + + public function testADetailPageIsMergedIntoItsRecordAndNotIndexedItself(): void + { + $this->client->method('manifest')->willReturn($this->manifest([ + $this->recordItem('product', 412, url: null), + $this->pageItem(7, detailOf: [['type' => 'product', 'uid' => 412]]), + ])); + $this->client->method('records')->willReturn([412 => ['uid' => 412, 'name' => 'Val Thorens', 'text' => []]]); + $this->client->method('page')->willReturn(PageFetchResult::ok('

Anreise

Mit dem Bus

')); + + $this->syncer()->run(); + + $record = $this->stored['product:412:de']; + $page = $this->stored['page:7:de']; + self::assertSame("# Val Thorens\n\n## Anreise\n\nMit dem Bus", $record->getBodyMd()); + self::assertSame('product:412', $page->getMergedInto()); + self::assertFalse($page->isIndexed()); + self::assertSame('https://www.ep-reisen.de/page-7', $record->getUrl(), 'a record without a page of its own is linkable through its detail page'); + self::assertSame('ep-reisen', $record->getSite()); + } + + public function testFailedAndEmptyPagesAreRememberedUntilTheyChange(): void + { + $this->client->method('manifest')->willReturn($this->manifest([$this->pageItem(1), $this->pageItem(2)])); + $this->client->method('page')->willReturnOnConsecutiveCalls(PageFetchResult::failed(404), PageFetchResult::ok('
')); + + $this->syncer()->run(); + + self::assertSame(ContentFetchState::Failed, $this->stored['page:1:de']->getFetchState()); + self::assertSame(ContentFetchState::Empty, $this->stored['page:2:de']->getFetchState()); + self::assertFalse($this->stored['page:1:de']->isStale()); + self::assertFalse($this->stored['page:2:de']->isStale()); + } + + public function testATimedOutPageIsRetriedNextRun(): void + { + $this->client->method('manifest')->willReturn($this->manifest([$this->pageItem(1)])); + $this->client->method('page')->willReturn(PageFetchResult::transient()); + + $report = $this->syncer()->run(); + + self::assertSame(1, $report->failed); + self::assertTrue($this->stored['page:1:de']->isStale()); + } + + public function testOutsideTheNightWindowAtMostFiftyPagesAreFetched(): void + { + $items = array_map(fn (int $uid): ManifestItem => $this->pageItem($uid), range(1, 60)); + $this->client->method('manifest')->willReturn($this->manifest($items)); + $this->client->method('page')->willReturn(PageFetchResult::ok('

Text

')); + + $report = $this->syncer('2026-10-08 12:00:00')->run(); + self::assertSame(50, $report->pages); + self::assertSame(10, $report->deferred); + } + + public function testInsideTheNightWindowEveryDuePageIsFetched(): void + { + $items = array_map(fn (int $uid): ManifestItem => $this->pageItem($uid), range(1, 60)); + $this->client->method('manifest')->willReturn($this->manifest($items)); + $this->client->method('page')->willReturn(PageFetchResult::ok('

Text

')); + + // 02:00 in Berlin. + $report = $this->syncer('2026-10-08 00:00:00')->run(); + + self::assertSame(60, $report->pages); + self::assertSame(0, $report->deferred); + } + + public function testAPausedFeedIsNotCalled(): void + { + $this->state->setPausedUntil(new \DateTimeImmutable('2026-10-08 13:00:00', new \DateTimeZone('UTC'))); + $client = $this->createMock(ContentFeedClient::class); + $client->expects(self::never())->method('manifest'); + $this->client = $client; + + $report = $this->syncer()->run(); + + self::assertNotNull($report->stoppedBy); + } + + private function syncer(string $now = '2026-10-08 12:00:00'): ContentSyncer + { + $repository = $this->createStub(ContentDocumentRepository::class); + $repository->method('findAllByKey')->willReturnCallback(fn (): array => $this->stored); + + $stateRepository = $this->createStub(ContentSyncStateRepository::class); + $stateRepository->method('get')->willReturn($this->state); + + $entityManager = $this->createStub(EntityManagerInterface::class); + $entityManager->method('persist')->willReturnCallback(function (object $entity): void { + if ($entity instanceof ContentDocument) { + $this->stored[$entity->getKey()] = $entity; + } + }); + $entityManager->method('remove')->willReturnCallback(function (object $entity): void { + if ($entity instanceof ContentDocument) { + unset($this->stored[$entity->getKey()]); + } + }); + $entityManager->method('contains')->willReturnCallback(fn (object $entity): bool => $entity instanceof ContentDocument && ($this->stored[$entity->getKey()] ?? null) === $entity); + + $converter = new HtmlToMarkdownConverter(); + + return new ContentSyncer( + $this->client, + $repository, + $stateRepository, + new DocumentBodyBuilder($converter), + $converter, + $this->createStub(TripCatalog::class), + $entityManager, + new MockClock(new \DateTimeImmutable($now, new \DateTimeZone('UTC'))), + new NullLogger(), + ); + } + + /** + * @param list $items + */ + private function manifest(array $items): ContentManifest + { + $keyed = []; + foreach ($items as $item) { + $keyed[$item->key()] = $item; + } + + return new ContentManifest(new \DateTimeImmutable('2026-10-08 08:15:00'), $keyed, ['www.ep-reisen.de']); + } + + /** + * @param list $detailOf + */ + private function pageItem(int $uid, array $detailOf = []): ManifestItem + { + return new ManifestItem('page', $uid, 'de', 'ep-reisen', 'https://www.ep-reisen.de/page-'.$uid, new \DateTimeImmutable('2026-09-30 12:00:00'), 'https://www.ep-reisen.de/page-'.$uid.'?type=1730', 'Page '.$uid, null, ['Skireisen'], $detailOf); + } + + private function recordItem(string $type, int $uid, ?string $url = 'https://www.ep-reisen.de/record'): ManifestItem + { + return new ManifestItem($type, $uid, 'de', null === $url ? null : 'ep-reisen', $url, new \DateTimeImmutable('2026-09-30 12:00:00')); + } + + private function document(string $type, int $uid): ContentDocument + { + return new ContentDocument($type, $uid, 'de', new \DateTimeImmutable('2026-09-01 12:00:00')); + } +} diff --git a/tests/ContentFeed/DocumentBodyBuilderTest.php b/tests/ContentFeed/DocumentBodyBuilderTest.php new file mode 100644 index 00000000..1d9db437 --- /dev/null +++ b/tests/ContentFeed/DocumentBodyBuilderTest.php @@ -0,0 +1,74 @@ +builder = new DocumentBodyBuilder(new HtmlToMarkdownConverter()); + } + + public function testProductSectionsInFixedOrderWithFaqs(): void + { + $body = $this->builder->build('product', [ + 'url' => 'https://www.ep-reisen.de/val-thorens/', + 'name' => 'Val Thorens Apartments', + 'subline' => 'Apartments', + 'headline' => 'Skiurlaub in Val Thorens', + 'text' => [ + 'skiPassDescription' => '

6 Tage 3 Vallées

', + 'teaser' => '

Ski in, ski out

', + 'includedServices' => '
  • Unterkunft
', + 'feature' => null, + ], + 'faqs' => [ + ['question' => 'Ist der Skipass inklusive?', 'answer' => '

Ja.

', 'category' => null], + ['question' => 'Ohne Antwort', 'answer' => null], + ], + ]); + + self::assertSame( + "# Val Thorens Apartments\n\nApartments\n\nSkiurlaub in Val Thorens\n\nSki in, ski out\n\n## Leistungen\n\n- Unterkunft\n\n## Skipass\n\n6 Tage 3 Vallées\n\n## Häufige Fragen\n\n### Ist der Skipass inklusive?\n\nJa.", + $body, + ); + } + + public function testHotelFactsAreSearchableText(): void + { + $body = $this->builder->build('hotel', [ + 'name' => 'Chalets de Rosael', + 'facts' => ['bedsCount' => 120, 'altitude' => null, 'sauna' => true, 'wifi' => false], + 'text' => ['description' => '

Ruhige Lage

'], + ]); + + self::assertSame("# Chalets de Rosael\n\n## Beschreibung\n\nRuhige Lage\n\n## Fakten\n\n- Betten: 120\n- Sauna: ja\n- WLAN: nein", $body); + } + + public function testNewsUsesTitleDateBodyAndTags(): void + { + $record = [ + 'title' => 'Packlisten', + 'teaser' => 'Was muss mit?', + 'bodytext' => '

Helm & Brille

', + 'datetime' => '2026-07-23T14:00:00+02:00', + 'tags' => ['Urlaubsvorbereitung', ['title' => 'Anfänger:in']], + ]; + + self::assertSame('Packlisten', $this->builder->title('news', $record)); + self::assertSame("# Packlisten\n\nVeröffentlicht am 23.07.2026\n\nWas muss mit?\n\nHelm & Brille\n\nThemen: Urlaubsvorbereitung, Anfänger:in", $this->builder->build('news', $record)); + } + + public function testRecordWithoutTextIsJustItsTitle(): void + { + self::assertSame('# Frankreich', $this->builder->build('country', ['name' => 'Frankreich', 'text' => []])); + } +} diff --git a/tests/ContentFeed/HtmlToMarkdownConverterTest.php b/tests/ContentFeed/HtmlToMarkdownConverterTest.php new file mode 100644 index 00000000..b11bc530 --- /dev/null +++ b/tests/ContentFeed/HtmlToMarkdownConverterTest.php @@ -0,0 +1,53 @@ +converter = new HtmlToMarkdownConverter(); + } + + public function testDropsEmbedsMediaAndFormsWithTheirContent(): void + { + $markdown = $this->converter->convert('

Text

BildIcon
'); + + self::assertSame('Text', $markdown); + } + + public function testKeepsHeadingsListsTablesAndEmphasis(): void + { + $markdown = $this->converter->convert('

Leistungen

  • Skipass
  • Bus
AB
12
'); + + self::assertSame("## Leistungen\n\n- **Skipass**\n- *Bus*\n\n| A | B |\n|---|---|\n| 1 | 2 |", $markdown); + } + + public function testResolvesRelativeLinksAgainstTheDocumentUrl(): void + { + $markdown = $this->converter->convert('AGB Kontakt Mail SZ', 'https://www.ep-reisen.de/info/reise'); + + self::assertSame('[AGB](https://www.ep-reisen.de/agb) [Kontakt](https://www.ep-reisen.de/info/kontakt) [Mail](mailto:info@ep-reisen.de) [SZ](https://snowzone.net/)', $markdown); + } + + public function testCollapsesTemplateWhitespaceAndDecodesEntities(): void + { + $markdown = $this->converter->convert("\n\n\n\n

Essen & Trinken

", 'https://www.ep-reisen.de/'); + + self::assertSame("[Ski- Gruppenhäuser](https://www.ep-reisen.de/x)\n\nEssen & Trinken", $markdown); + } + + public function testEmptyInputIsEmpty(): void + { + self::assertSame('', $this->converter->convert(null)); + self::assertSame('', $this->converter->convert(' ')); + self::assertSame('', $this->converter->convert('
')); + } +} diff --git a/tests/Security/McpAccessTokenHandlerTest.php b/tests/Security/McpAccessTokenHandlerTest.php new file mode 100644 index 00000000..1f35b6f5 --- /dev/null +++ b/tests/Security/McpAccessTokenHandlerTest.php @@ -0,0 +1,31 @@ +getUserBadgeFrom('s3cret')->getUserIdentifier()); + } + + public function testAWrongTokenIsRejected(): void + { + $this->expectException(BadCredentialsException::class); + + (new McpAccessTokenHandler('s3cret'))->getUserBadgeFrom('guess'); + } + + public function testAnUnconfiguredTokenRejectsEverything(): void + { + $this->expectException(BadCredentialsException::class); + + (new McpAccessTokenHandler(''))->getUserBadgeFrom(''); + } +}