diff --git a/.ddev/commands/web/static-export b/.ddev/commands/web/static-export new file mode 100755 index 00000000..9f8e0390 --- /dev/null +++ b/.ddev/commands/web/static-export @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +## Description: Export the TYPO3 sites as static document roots (no PHP) +## Usage: static-export [--site ] [--url ] [--no-follow] [--out ] [--workers ] [--hybrid --typo3-current [--live-ajax]] +## Example: "ddev static-export --site ep-reisen" +set -euo pipefail + +exec python3 -I "${DDEV_APPROOT}/.ddev/static-export/export.py" "$@" diff --git a/.ddev/static-export/__pycache__/export.cpython-313.pyc b/.ddev/static-export/__pycache__/export.cpython-313.pyc new file mode 100644 index 00000000..e967ee47 Binary files /dev/null and b/.ddev/static-export/__pycache__/export.cpython-313.pyc differ diff --git a/.ddev/static-export/checks/check_live.py b/.ddev/static-export/checks/check_live.py new file mode 100644 index 00000000..9b172069 --- /dev/null +++ b/.ddev/static-export/checks/check_live.py @@ -0,0 +1,36 @@ +"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files.""" +import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections +root, prod = sys.argv[1], sys.argv[2].rstrip('/') +host = urllib.parse.urlsplit(prod).netloc +class NoRedir(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a): return None +op = urllib.request.build_opener(NoRedir) +def get(url, body=False): + try: + with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r: + return r.status, (r.read().decode('utf-8', 'replace') if body else '') + except urllib.error.HTTPError as e: return e.code, '' + except Exception as e: return str(e)[:60], '' +pages = [] +for d, _, files in os.walk(root): + if '/_ajax' in d: continue + for f in files: + if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else '')) +pages = sorted(set(p.replace('//', '/') for p in pages)) +with concurrent.futures.ThreadPoolExecutor(12) as ex: + res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages))) +refs = set() +for p, (st, body) in res.items(): + for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body): + for c in html.unescape(v).split(','): + c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c) + if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'): + base = 'https://' + (u.netloc or host) + refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else '')) +with concurrent.futures.ThreadPoolExecutor(16) as ex: + rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs)))) +print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values())))) +by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items()) +print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)] +bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))] +for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u)) diff --git a/.ddev/static-export/checks/check_site.py b/.ddev/static-export/checks/check_site.py new file mode 100644 index 00000000..11c29117 --- /dev/null +++ b/.ddev/static-export/checks/check_site.py @@ -0,0 +1,30 @@ +"""Request every exported page and every local URL referenced in pages/fragments via a local Apache.""" +import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures +root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099' +class NoRedir(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a): return None +op = urllib.request.build_opener(NoRedir) +def get(path): + try: + with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status + except urllib.error.HTTPError as e: return e.code + except Exception as e: return str(e) +pages, refs = [], set() +for d, _, files in os.walk(root): + for f in files: + if not f.endswith('.html'): continue + rel = '/' + os.path.relpath(os.path.join(d, f), root) + if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel) + t = open(os.path.join(d, f), encoding='utf-8').read() + for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t): + for c in html.unescape(v).split(','): + c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c) + if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'): + refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else '')) +with concurrent.futures.ThreadPoolExecutor(16) as ex: + pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs)))) +bad_p = {k: v for k, v in pr.items() if v != 200} +bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)} +print('pages: %d, not 200: %d' % (len(pr), len(bad_p))) +print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r))) +for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k)) diff --git a/.ddev/static-export/checks/compare_api.py b/.ddev/static-export/checks/compare_api.py new file mode 100644 index 00000000..5e8adb47 --- /dev/null +++ b/.ddev/static-export/checks/compare_api.py @@ -0,0 +1,41 @@ +"""Compare /api/hotel + /api/product of a static export (Apache on 127.0.0.1:8099) with live local TYPO3. + +Usage: compare_api.py . Case list: every hotel/product code, sampled pairs, random +unlinked pairs, empty/missing params, missing/wrong/query-string key, direct snapshot access. +""" +import sys, os, random, importlib.util, urllib.request, urllib.error, concurrent.futures +spec = importlib.util.spec_from_file_location('export', '/var/www/html/.ddev/static-export/export.py'); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +sites = m.load_sites(); exp = m.Exporter('ep-reisen', sites['ep-reisen'], sites, '/nonexistent', follow=False, workers=1); key = m.cms_api_key() +class N(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a): return None +op = urllib.request.build_opener(N) +def call(url, headers): + try: + with op.open(urllib.request.Request(url, headers=headers), timeout=60) as r: return r.status, r.read().decode(), r.headers.get('Content-Type') + except urllib.error.HTTPError as e: return e.code, e.read().decode(), e.headers.get('Content-Type') +typo3 = lambda q, h: call('http://127.0.0.1/' + q, dict(h, **{'Host': 'ep-reisen.ddev.site', 'X-Forwarded-Proto': 'https'})) +static = lambda q, h: call('http://127.0.0.1:8099/' + q, dict(h, **{'Host': 'www.ep-reisen.de'})) +root = sys.argv[1]; K = {'X-Api-Key': key} +hotels = sorted(f[:-5] for f in os.listdir(root + '/_api/hotel')); products = sorted(f[:-5] for f in os.listdir(root + '/_api/product') if f.endswith('.json')) +pairs = sorted((p, f[:-5]) for p in products if os.path.isdir(root + '/_api/product/' + p) for f in os.listdir(root + '/_api/product/' + p) if not f.startswith('_')) +random.seed(1) +cases = [('api/hotel?hotel=' + c, K) for c in hotels] + [('api/product?product=' + c, K) for c in products] +cases += [('api/product?product=%s&hotel=%s' % pc, K) for pc in random.sample(pairs, 150)] +cases += [('api/product?hotel=%s&product=%s' % (h, p), K) for p, h in random.sample(pairs, 10)] +cases += [('api/product?product=%s&hotel=%s' % (p, random.choice(hotels)), K) for p in random.sample(products, 60)] +cases += [(q, K) for q in ['api/hotel?hotel=_empty', 'api/product?product=_empty', 'api/hotel?hotel=%40empty', 'api/product?product=3VIIN&hotel=@empty', 'api/product?product=3VIIN&hotel=PMVSW%26A', 'api/hotel?hotel=a%20b', 'api/product?product=%40empty&hotel=', 'api/hotel', 'api/product', 'api/hotel?hotel=', 'api/product?product=', 'api/product?product=&hotel=X', + 'api/hotel?hotel=NOPE', 'api/product?product=NOPE', 'api/product?product=NOPE&hotel=' + hotels[0], + 'api/product?product=%s&hotel=' % products[0], 'api/product?product=%s&hotel=NOPE' % products[0]]] +cases += [('api/hotel?hotel=' + hotels[0], {}), ('api/hotel?hotel=' + hotels[0], {'X-Api-Key': 'wrong'}), + ('api/hotel?hotel=%s&key=%s' % (hotels[0], key), {}), ('api/product?key=%s&product=%s' % (key, products[0]), {}), + ('api/product?product=%s&key=wrong' % products[0], {})] +def compare(c): + q, h = c; a, b = typo3(q, h), static(q, h) + return q, (a[0], exp.to_prod(a[1]), a[2]), b +bad = [] +with concurrent.futures.ThreadPoolExecutor(8) as ex: + for q, a, b in ex.map(compare, cases): + if a != b: bad.append((q, a, b)) +print('cases: %d, identical: %d, different: %d' % (len(cases), len(cases) - len(bad), len(bad))) +for q, a, b in bad[:6]: print(' DIFF', q, '| typo3', a[0], a[2], '| static', b[0], b[2], repr(b[1][:90])) +for q in ['_api/hotel/%s.json' % hotels[0], '_api/pair.shtml', '_api/product-part/%s.part' % products[0]]: print(' direct', q, static(q, {})[0]) diff --git a/.ddev/static-export/checks/import_compare.php b/.ddev/static-export/checks/import_compare.php new file mode 100644 index 00000000..c0c9da9a --- /dev/null +++ b/.ddev/static-export/checks/import_compare.php @@ -0,0 +1,64 @@ +_. The live tables are backed up first and restored at the end. + */ +putenv('TYPO3_PATH_ROOT=/var/www/html/public'); +putenv('TYPO3_PATH_APP=/var/www/html'); +$classLoader = require '/var/www/html/vendor/autoload.php'; +\TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(0, \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::REQUESTTYPE_CLI); +\TYPO3\CMS\Core\Core\Bootstrap::init($classLoader); + +use EP\EpProducts\Service\ApiImportSource; +use EP\EpProducts\Service\DateImportService; +use EP\EpProducts\Service\XmlProductReader; +use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Utility\GeneralUtility; + +$db = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default'); +$tables = ['date' => 'tx_epproducts_domain_model_date', 'room' => 'tx_epproducts_domain_model_room', 'product' => 'tx_epproducts_domain_model_product']; + +$snapshot = function (string $prefix) use ($db, $tables) { + foreach ($tables as $short => $table) { + $db->executeStatement("DROP TABLE IF EXISTS cmp_{$prefix}_{$short}"); + $db->executeStatement("CREATE TABLE cmp_{$prefix}_{$short} AS SELECT * FROM {$table}"); + } +}; +$restore = function () use ($db, $tables) { + foreach ($tables as $short => $table) { + $db->executeStatement("DELETE FROM {$table}"); + $db->executeStatement("INSERT INTO {$table} SELECT * FROM cmp_bak_{$short}"); + } +}; + +$snapshot('bak'); +try { + $reader = new XmlProductReader(GeneralUtility::getFileAbsFileName('fileadmin/xmlexport')); + $rows = iterator_to_array($reader, false); + $count = GeneralUtility::makeInstance(DateImportService::class)->import($rows); + printf("xml: %d rows, %d dates imported, source generated at %s\n", count($rows), $count, + $reader->getSourceGeneratedAt() ? $reader->getSourceGeneratedAt()->format('c') : '?'); + $snapshot('xml'); + $restore(); + + // Bypass the "already imported" baseline: walk the export pages directly + $api = GeneralUtility::makeInstance(ApiImportSource::class); + $walk = new \ReflectionMethod($api, 'walkPages'); + $walk->setAccessible(true); + $export = $walk->invoke($api); + if (!is_array($export)) { + throw new \RuntimeException('MyE&P export walk returned ' . var_export($export, true)); + } + [$apiRows, $generatedAt] = $export; + $count = GeneralUtility::makeInstance(DateImportService::class)->import($apiRows); + printf("api: %d rows, %d dates imported, source generated at %s\n", count($apiRows), $count, + $generatedAt ? $generatedAt->format('c') : '?'); + $snapshot('api'); + + // Raw rows of both sources, for a field-by-field comparison independent of the import + file_put_contents('/var/www/html/var/rows_xml.json', json_encode($rows)); + file_put_contents('/var/www/html/var/rows_api.json', json_encode($apiRows)); +} finally { + $restore(); + echo "live tables restored\n"; +} diff --git a/.ddev/static-export/export.py b/.ddev/static-export/export.py new file mode 100644 index 00000000..c29fa5fb --- /dev/null +++ b/.ddev/static-export/export.py @@ -0,0 +1,936 @@ +#!/usr/bin/env python3 +"""Static export of the TYPO3 sites running in DDEV. + +Crawls each site (sitemap + internal links), pre-fetches the AJAX dates and +price tables of product pages, rewrites *.ddev.site hosts to the production +domains and copies every referenced static asset. The result is a set of +plain document roots that run without PHP. + +With --hybrid the document roots sit in front of TYPO3 13 instead: product pages +and the search are left out (TYPO3 13 serves them through index.php), the search +UI stays (except the search bar), and the API is answered live. + +Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR] + [--hybrid --typo3-current PATH [--live-ajax]] +""" + +import argparse +import collections +import concurrent.futures +import hashlib +import html +import http.client +import os +import re +import shutil +import subprocess +import sys +import threading +import urllib.error +import urllib.parse +import urllib.request + +APP_ROOT = os.environ.get('DDEV_APPROOT', '/var/www/html') +PUBLIC = os.path.join(APP_ROOT, 'public') +SITES_DIR = os.path.join(APP_ROOT, 'config', 'sites') + +# Hosts that pull from the production origin; their paths must exist in the export +CDN_HOSTS = {'cdn.ep-reisen.de'} + +# Any attribute pointing at TYPO3's AJAX page type (product tables, HTMX teaser +# popups, ...) is pre-fetched; JS may append "&pa=..." to these URLs +AJAX_PATH = '/ajax/' + +# Forms posting to these TYPO3 plugins (contact, enquiry, application, booking, newsletter) +# cannot work statically and are replaced by a contact notice +CONTACT_FORM_RE = re.compile(r']*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I) + +# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them, +# and links to the search page (menu icon, "Alle anzeigen" with preset filters) +SEARCH_ELEMENTS = ( + ('div', re.compile(r']*\bdata-controller="searchbar"[^>]*>')), + ('div', re.compile(r']*\bdata-controller="mobile-search"[^>]*>')), + ('button', re.compile(r']*\bdata-overlay-toggle-id-value="(?:mobile-)?search"[^>]*>')), + ('a', re.compile(r']*\bhref="[^"]*/suche/(?:\?[^"]*)?"[^>]*>')), +) +SEARCH_PAGE_RE = re.compile(r'/suche/$') + +# Hybrid: only the search bar goes (TYPO3 13 has no search bar plugin); links, menu icons and +# the mobile filter overlay stay, the search itself is served by TYPO3 13 +HYBRID_SEARCH_ELEMENTS = ( + ('div', re.compile(r']*\bdata-controller="searchbar"[^>]*>')), + ('button', re.compile(r']*\bdata-controller="searchbar-toggle"[^>]*>')), +) +# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on +# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl +# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash +# (404 on TYPO3 13 for popups pointing at another site) +LIVE_AJAX = { + ('AjaxTable', 'datestable'), ('AjaxTable', 'pricetable'), ('AjaxTable', 'pricetableHtml'), + ('AjaxTable', 'eventPricetable'), + ('AjaxCalendar', 'priceTable'), ('AjaxCalendar', 'priceCalendar'), + ('AjaxSearch', 'searchresult'), ('AjaxFilterPanel', 'index'), ('AjaxWatchlist', 'list'), +} +LIVE_AJAX_PARAM_RE = re.compile(r'tx_epproducts_ajax(?:%5B|\[)(controller|action)(?:%5D|\])=(\w+)') + +# Contact shown instead of forms: display phone, tel: number, e-mail (as in each site's footer) +CONTACTS = { + 'ep-reisen': ('0221 272276-0', '+492212722760', 'kundenservice@ep-reisen.de'), + 'ski-boarderweek': ('0221 272276-47', '+4922127227647', 'eventreisen@ep-reisen.de'), + 'snowzone': ('0221 272276-47', '+4922127227647', 'snowzone@ep-reisen.de'), + 'unichamp': ('0221 272276-47', '+4922127227647', 'eventreisen@ep-reisen.de'), + 'ser': ('0221 272276-41', '+4922127227641', 'info@semesterende-skireisen.de'), + 'alvac': ('0221 272276-30', '+4922127227630', 'info@alpinevacation.ch'), + 'cllt': ('0221 272276-30', '+4922127227630', 'info@cllt-touristik.at'), +} +DEFAULT_CONTACT = CONTACTS['ep-reisen'] + +CONTACT_TEXT = { + 'de': ('Dieses Formular ist vorübergehend nicht verfügbar.', + 'Du hast Fragen oder möchtest buchen? Unser Kundenservice hilft dir gerne weiter:'), + 'en': ('This form is temporarily unavailable.', + 'Questions or want to book? Our customer service is happy to help:'), +} + +CONTACT_NOTICE = '''
+

{lead}

+

{text}

+

+{phone} +{email} +

+
''' + + +def replace_elements(text, tag, start_re, replacement): + """Replace each element whose start tag matches, including its balanced content.""" + token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I) + out, pos, count = [], 0, 0 + for m in start_re.finditer(text): + if m.start() < pos: + continue # nested inside an element already replaced + depth, end = 1, None + for t in token_re.finditer(text, m.end()): + depth += -1 if t.group(1) else 1 + if depth == 0: + end = t.end() + break + if end is None: + continue # unbalanced markup, leave it alone + out += [text[pos:m.start()], replacement] + pos, count = end, count + 1 + out.append(text[pos:]) + return ''.join(out), count + +URL_ATTR_RE = re.compile(r'(?:href|src|srcset|data-src|data-srcset|content|poster|action)="([^"]*)"') +CSS_URL_RE = re.compile(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)') +ASSET_EXT_RE = re.compile(r'\.(css|js|mjs|json|map|svg|png|jpe?g|gif|webp|avif|ico|woff2?|ttf|eot|otf|pdf|mp4|webm|txt|xml|webmanifest)$', re.I) + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Report redirects instead of following them, so they become rewrite rules.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +OPENER = urllib.request.build_opener(NoRedirect) + + +def rewrite_rule(path, location, status): + pattern = re.escape(urllib.parse.unquote(path).strip('/')).replace('\\ ', '\\x20') + target = re.sub(r'([$%])', r'\\\1', location) + return '\tRewriteRule "^%s/?$" "%s" [R=%d,L,NE]' % (pattern, target, status) + + +def load_sites(only=None): + """Read base and Development base variant from each site config.""" + sites = {} + for name in sorted(os.listdir(SITES_DIR)): + path = os.path.join(SITES_DIR, name, 'config.yaml') + if not os.path.isfile(path) or (only and name not in only): + continue + with open(path, encoding='utf-8') as fh: + text = fh.read() + prod = re.search(r"^base:\s*'?([^'\s]+)", text, re.M) + dev = re.search(r"base:\s*'?(https://[^'\s]+\.ddev\.site)'?\s*\n\s*condition:\s*'applicationContext == \"Development\"'", text) + if prod and dev and prod.group(1).startswith('http'): + root = re.search(r'^rootPageId:\s*(\d+)', text, re.M) + sites[name] = {'prod': prod.group(1).rstrip('/'), 'dev': dev.group(1).rstrip('/'), + 'root': int(root.group(1)) if root else 0, 'languages': site_languages(text)} + return sites + + +def site_languages(text): + """languageId => base path prefix ('' for '/', '/en' for '/en/') of a site config.""" + section = re.search(r'^languages:\n((?:[ \t-].*\n|\n)*)', text, re.M) + languages = {} + for item in re.split(r'^ -[ \t]*\n?', section.group(1) if section else '', flags=re.M): + language_id = re.search(r"^\s*languageId:\s*'?(\d+)", item, re.M) + base = re.search(r"^\s{4}base:\s*'?([^'\s]+)", item, re.M) + if language_id and base and base.group(1).startswith('/'): + languages[int(language_id.group(1))] = base.group(1).rstrip('/') + return languages + + +class Exporter: + def __init__(self, name, site, all_sites, out_dir, follow, workers, hybrid=False, live_ajax=False): + self.site = site + self.hybrid = hybrid + self.live_ajax = live_ajax + self.dynamic = dynamic_paths(site) if hybrid else set() + self.route_prefixes = set(route_pages(site).values()) if hybrid else set() + self.contact = CONTACTS.get(name, DEFAULT_CONTACT) + self.dev = site['dev'] + self.dev_host = urllib.parse.urlsplit(self.dev).netloc + # Editors often paste absolute production URLs; those are internal links too + self.prod_host = urllib.parse.urlsplit(site['prod']).netloc + self.ajax_results = {} + self.ajax_locks = {} + self.out = out_dir + self.follow = follow + self.workers = workers + # dev → prod for every site, so cross-site links come out right too + self.host_map = [(s['dev'], s['prod']) for s in all_sites.values()] + # Some pages get their AJAX URLs below a parent slug (/reisen-fuer-gruppen/ajax/html?...) + self.ajax_attr_re = re.compile(r'([a-zA-Z][\w:.-]*)="((?:%s)?(?:/[\w-]+)*%s[^"]*)"' % (re.escape(self.dev), re.escape(AJAX_PATH))) + self.seen = set() + self.assets = set() + self.lock = threading.Lock() + self.errors = [] + self.crawl_redirects = [] + self.follow_redirects = follow + self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'search': 0, 'dynamic': 0} + + # --- helpers ----------------------------------------------------------- + + def fetch(self, url, extra_headers=None, log_errors=True): + """Return status, body and (for redirects) the Location header.""" + headers = {'User-Agent': 'ep-static-export'} + headers.update(extra_headers or {}) + parts = urllib.parse.urlsplit(url) + if parts.scheme == 'https' and parts.hostname.endswith('.ddev.site'): + # Talk to the container's own web server, bypassing ddev-router (works for hostnames + # added without "ddev restart"); TYPO3 still sees HTTPS via the forwarded header + url = urllib.parse.urlunsplit(('http', '127.0.0.1') + tuple(parts[2:])) + headers.update({'Host': parts.hostname, 'X-Forwarded-Proto': 'https'}) + req = urllib.request.Request(url, headers=headers) + try: + with OPENER.open(req, timeout=60) as res: + return res.status, res.read().decode('utf-8', 'replace'), None + except urllib.error.HTTPError as e: + return e.code, e.read().decode('utf-8', 'replace'), e.headers.get('Location') + except Exception as e: # noqa: BLE001 - report and continue + if log_errors: + self.error('%s: %s' % (url, e)) + return 0, '', None + + def error(self, msg): + with self.lock: + self.errors.append(msg) + + def write(self, rel_path, content): + path = os.path.join(self.out, rel_path.lstrip('/')) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf-8') as fh: + fh.write(content) + + def to_prod(self, text): + for dev, prod in self.host_map: + text = text.replace(dev, prod) + text = text.replace(dev.replace('/', '\\/'), prod.replace('/', '\\/')) + text = text.replace(urllib.parse.quote(dev, safe=''), urllib.parse.quote(prod, safe='')) + return text + + def local_path(self, url): + """Path on this site for a URL, or None if it points elsewhere.""" + parts = urllib.parse.urlsplit(html.unescape(url)) + if parts.scheme and parts.scheme not in ('http', 'https'): + return None + if parts.netloc and parts.netloc not in (self.dev_host, self.prod_host) and parts.netloc not in CDN_HOSTS: + return None + if not parts.path.startswith('/') or parts.path.startswith('//'): + return None + return parts + + def collect(self, text): + """Collect asset paths and (if following) internal page links.""" + links = set() + for raw in URL_ATTR_RE.findall(text) + CSS_URL_RE.findall(text): + for candidate in raw.split(','): # srcset + candidate = candidate.strip().split(' ')[0] + parts = self.local_path(candidate) + if not parts: + continue + path = urllib.parse.unquote(parts.path) + if ASSET_EXT_RE.search(path): + self.assets.add(path) + elif self.follow and not parts.query and parts.netloc in ('', self.dev_host, self.prod_host) \ + and not path.startswith(('/typo3', AJAX_PATH)): + links.add(self.dev + path) + return links + + # --- pages and AJAX tables --------------------------------------------- + + def ajax_file(self, url): + """Fetch an AJAX table, store it, return its static URL.""" + url = html.unescape(url) + if url.startswith('/'): + url = self.dev + url + name = '/_ajax/%s.html' % hashlib.sha1(url.replace(self.dev, '').encode()).hexdigest() + with self.lock: + name_lock = self.ajax_locks.setdefault(name, threading.RLock()) + # Every reference waits for the first fetch, so a failed table keeps its original URL everywhere + with name_lock: + if name not in self.ajax_results: + # Set before fetching: a response referencing itself must not recurse. + # The JS appends "&pa=..."; the dummy query keeps that harmless + self.ajax_results[name] = name + '?s=1' + status, body, _ = self.fetch(url) + if status != 200: + self.error('%s → %s' % (url, status)) + self.ajax_results[name] = url + else: + self.write(name, self.to_prod(self.process(self.strip_dynamic(body, '')))) + with self.lock: + self.stats['ajax'] += 1 + return self.ajax_results[name] + + def strip_dynamic(self, text, path): + """Swap contact forms for a contact notice and drop the search UI.""" + phone, tel, email = self.contact + lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de'] + notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email) + text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice) + removed = 0 + for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS): + text, n = replace_elements(text, tag, start_re, '') + removed += n + with self.lock: + self.stats['forms'] += forms + self.stats['search'] += removed + return text + + def process(self, text): + """Replace AJAX table URLs (recursively fetching them) and collect assets.""" + text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else + '%s="%s"' % (m.group(1), html.escape(self.ajax_file(m.group(2)))), text) + self.collect(text) + return text + + def is_live_ajax(self, url): + """Hybrid --live-ajax: keep the URL when TYPO3 13 serves the action.""" + params = dict(LIVE_AJAX_PARAM_RE.findall(html.unescape(url))) + return self.live_ajax and (params.get('controller'), params.get('action')) in LIVE_AJAX + + def is_dynamic(self, path): + """Hybrid: product pages (with their /detail/ variants) and search pages belong to TYPO3 13.""" + path = path.rstrip('/') + '/' + return (path in self.dynamic or any(path.startswith(p + 'detail/') for p in self.dynamic) + or any(path.startswith(p) and path != p for p in self.route_prefixes)) + + def safe_page(self, url): + try: + return self.page(url) + except Exception as e: # noqa: BLE001 - one broken page must not stop the export + self.error('%s: %s: %s' % (url, type(e).__name__, e)) + return set() + + def page(self, url): + status, body, location = self.fetch(url) + if status in (301, 302, 303, 307, 308) and location: + location = urllib.parse.urljoin(url, location) + with self.lock: + self.crawl_redirects.append(rewrite_rule(urllib.parse.urlsplit(url).path, self.to_prod(location), status)) + internal = location.startswith(self.dev + '/') and '?' not in location + return {location} if internal and self.follow_redirects else set() + if status != 200: + self.error('%s → %s' % (url, status)) + return set() + path = urllib.parse.urlsplit(url).path + if self.hybrid and self.is_dynamic(path): + # Served by TYPO3 13 through index.php; only its links are followed + with self.lock: + self.stats['dynamic'] += 1 + return self.collect(body) + if not self.hybrid and SEARCH_PAGE_RE.search(path): + # Search does not work statically: send visitors to the homepage instead + with self.lock: + self.crawl_redirects.append(rewrite_rule(path, self.site['prod'] + '/', 302)) + return set() + # Extensionless paths become directories, so /a and /a/b/ can both exist + target = path if re.search(r'\.[a-z0-9]+$', path, re.I) else path.rstrip('/') + '/index.html' + body = self.strip_dynamic(body, path) + links = self.collect(body) + self.write(target, self.to_prod(self.process(body))) + with self.lock: + self.stats['pages'] += 1 + return links + + def sitemap(self): + """Export sitemap index and children as plain files, return page URLs.""" + status, index, _ = self.fetch(self.dev + '/sitemap.xml') + if status != 200: + print(' note: no sitemap.xml (%s), crawling from / only' % status) + return [] + urls = [] + for loc in re.findall(r'([^<]+)', index): + child_url = html.unescape(loc) + name = urllib.parse.parse_qs(urllib.parse.urlsplit(child_url).query).get('sitemap', ['pages'])[0] + _, child, _ = self.fetch(child_url) + urls += [html.unescape(u) for u in re.findall(r'([^<]+)', child)] + self.write('/sitemap-%s.xml' % name, self.to_prod(child)) + index = index.replace(loc, self.dev + '/sitemap-%s.xml' % name) + self.write('/sitemap.xml', self.to_prod(index)) + return [u for u in urls if u.startswith(self.dev)] + + def crawl(self, start): + queue = list(dict.fromkeys(start)) + self.seen.update(queue) + with concurrent.futures.ThreadPoolExecutor(self.workers) as pool: + while queue: + batch, queue = queue, [] + for links in pool.map(self.safe_page, batch): + for link in links: + if link not in self.seen: + self.seen.add(link) + queue.append(link) + + def redirects(self): + """Backend redirects (sys_redirect) as RewriteRules, targets resolved by TYPO3 itself.""" + host = urllib.parse.urlsplit(self.site['prod']).netloc + where = "deleted=0 AND disabled=0 AND source_host IN ('%s', '*')" % host + try: + out = subprocess.run( + ['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', + 'SELECT source_path FROM sys_redirect WHERE %s AND is_regexp=0; ' + 'SELECT COUNT(*) FROM sys_redirect WHERE %s AND is_regexp=1' % (where, where)], + capture_output=True, text=True, check=True).stdout.splitlines() + except (OSError, subprocess.CalledProcessError) as e: + self.error('sys_redirect: %s' % e) + return '' + paths, regexp = sorted(set(out[:-1])), int(out[-1]) + skipped = ['regexp redirects (not exported): %d' % regexp] if regexp else [] + + def resolve(path): + if '?' in path: + return path, 0, 'query string in source path' + conn = http.client.HTTPConnection('127.0.0.1', 80, timeout=60) + try: + conn.request('GET', urllib.parse.quote(path, safe="/%:@!$&'()*+,;=-._~"), headers={'Host': host}) + res = conn.getresponse() + res.read() + return path, res.status, res.getheader('Location') + finally: + conn.close() + + rules = ['\t# Redirects from the TYPO3 backend (sys_redirect), resolved by TYPO3 at export time'] + with concurrent.futures.ThreadPoolExecutor(self.workers) as pool: + for path, status, location in pool.map(resolve, paths): + if status not in (301, 302, 303, 307, 308) or not location: + # Mostly dead entries (target page gone) that 404 in production already + skipped.append('%s → %s %s' % (path, status, location or '')) + continue + # Resolve like a browser: TYPO3 sends "https:/path" for relative targets + location = urllib.parse.urljoin(self.dev + path, location) + if location.rstrip('/') == (self.dev + path).rstrip('/'): + # TYPO3 matches "/a" for "/a/" too, so a redirect to the page at "/a/" loops + skipped.append('%s → %s (redirects to itself)' % (path, location)) + continue + target = urllib.parse.urlsplit(location) + if target.netloc == self.dev_host and ASSET_EXT_RE.search(target.path): + self.assets.add(urllib.parse.unquote(target.path)) + rules.append(rewrite_rule(path, self.to_prod(location), status)) + self.stats['redirects'] = len(rules) - 1 + self.stats['redirects_skipped'] = len(skipped) + with open(self.out.rstrip('/') + '-redirects-skipped.txt', 'w', encoding='utf-8') as fh: + fh.write('\n'.join(skipped) + '\n') + return '\n'.join(rules) + '\n' + + def special_files(self): + status, body, _ = self.fetch(self.dev + '/robots.txt') + if status == 200: + self.write('/robots.txt', self.to_prod(body)) + # TYPO3 error page, served by Apache via ErrorDocument 404 + status, body, _ = self.fetch(self.dev + '/static-export-not-found/') + if status == 404 and body: + self.write('/404.html', self.to_prod(self.process(self.strip_dynamic(body, '/')))) + + # --- static assets ----------------------------------------------------- + + def copy_assets(self): + pending, copied, missing = list(self.assets), set(), [] + while pending: + path = pending.pop() + if path in copied: + continue + copied.add(path) + src = os.path.join(PUBLIC, path.lstrip('/')) + if not os.path.isfile(src): + missing.append(path) + continue + dst = os.path.join(self.out, path.lstrip('/')) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + if path.endswith('.css'): # fonts and images referenced from CSS + with open(src, encoding='utf-8', errors='replace') as fh: + for ref in CSS_URL_RE.findall(fh.read()): + if ref.startswith(('data:', 'http', '#')): + continue + ref = urllib.parse.urlsplit(ref).path + pending.append(os.path.normpath(os.path.join(os.path.dirname(path), ref))) + return len(copied) - len(missing), missing + + +STATIC_HEADER = '''### Begin: Static export ### +# Generated by .ddev/static-export/export.py from public/.htaccess. +# No PHP: TYPO3's index.php catch-all and StaticFileCache rules are removed. +DirectoryIndex index.html +ErrorDocument 404 /404.html +Options -Indexes -MultiViews + + php_flag engine off + +### End: Static export ### + +''' + +# --- TYPO3 backchannel API (/api/hotel, /api/product) ---------------------------------------- + +# MyE&P calls the API on this site; ApiMiddleware itself answers on every host +API_SITES = {'ep-reisen'} +API_CODE_RE = re.compile(r'^[A-Za-z0-9_-]+$') + + +def mysql_rows(sql): + out = subprocess.run(['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', sql], + capture_output=True, text=True, check=True).stdout + return [line.split('\t') for line in out.splitlines() if line] + + +# Records that only exist as URL arguments of one page (route enhancers), not linked anywhere: +# (page uid, table, field, pattern the value must match or None) +ROUTE_RECORDS = ( + (1846, 'tx_epproducts_domain_model_product', 'path_segment', None), # /gruppe/{product} + (2736, 'tx_epproducts_domain_model_travelinfo', 'path_segment', None), # /reiseinformationen/{name} + (2736, 'tx_epproducts_domain_model_travelinfo', 'travel_code', r'[A-Z]+\d{6}'), # /reiseinformationen/{code} +) + + +def page_tree_check(site): + """Return a function telling whether a default-language page uid belongs to the site.""" + parents = {int(uid): int(pid) for uid, pid in mysql_rows('SELECT uid, pid FROM pages WHERE sys_language_uid = 0')} + + def in_site(uid): + for _ in range(100): + if uid == site['root']: + return True + if uid not in parents: + return False + uid = parents[uid] + return False + return in_site + + +def route_pages(site): + """Slug (with trailing slash) of each route page of ROUTE_RECORDS on this site, by page uid.""" + in_site = page_tree_check(site) + base = urllib.parse.urlsplit(site['dev']).path.rstrip('/') + pages = {} + for uid in sorted({r[0] for r in ROUTE_RECORDS}): + rows = mysql_rows('SELECT slug FROM pages WHERE uid = %d AND deleted = 0 AND hidden = 0' % uid) + if rows and in_site(uid): + pages[uid] = base + rows[0][0].rstrip('/') + '/' + return pages + + +def route_urls(site): + """Start URLs for the records of ROUTE_RECORDS (not reachable through links or the sitemap).""" + pages = route_pages(site) + urls = set() + for uid, table, field, pattern in ROUTE_RECORDS: + if uid not in pages: + continue + for (value,) in mysql_rows("SELECT DISTINCT %s FROM %s WHERE deleted = 0 AND hidden = 0 AND %s <> ''" % (field, table, field)): + if pattern is None or re.fullmatch(pattern, value): + urls.add(site['dev'].rstrip('/') + pages[uid] + urllib.parse.quote(value) + '/') + return sorted(urls) + + +def dynamic_paths(site): + """Paths (with trailing slash) of the pages TYPO3 13 serves in hybrid mode: pages of this site + holding the product detail or search result plugin, in every language of the site.""" + in_site = page_tree_check(site) + rows = mysql_rows( + "SELECT DISTINCT p.slug, p.sys_language_uid, IF(p.sys_language_uid = 0, p.uid, p.l10n_parent) " + "FROM pages p JOIN tt_content c ON c.pid = p.uid " + "WHERE p.deleted = 0 AND p.hidden = 0 AND c.deleted = 0 AND c.hidden = 0 AND c.CType = 'list' " + "AND c.list_type IN ('epproducts_product_detail', 'epproducts_searchresult')") + base = urllib.parse.urlsplit(site['dev']).path.rstrip('/') + paths = set() + for slug, language, default_uid in rows: + if int(language) in site['languages'] and in_site(int(default_uid)): + paths.add(base + site['languages'][int(language)] + slug.rstrip('/') + '/') + return paths + + +FRONT_CONTROLLER_STUB = """/...) come from the release as well: Apache does not +// follow a symlink from the docroot into the release on every vhost, PHP does. +$path = rawurldecode((string)strtok($_SERVER['REQUEST_URI'] ?? '', '?')); +if (str_starts_with($path, '/_assets/')) { + $types = [ + 'css' => 'text/css; charset=utf-8', 'js' => 'text/javascript; charset=utf-8', 'json' => 'application/json', + 'map' => 'application/json', 'svg' => 'image/svg+xml', 'png' => 'image/png', 'jpg' => 'image/jpeg', + 'jpeg' => 'image/jpeg', 'gif' => 'image/gif', 'webp' => 'image/webp', 'avif' => 'image/avif', + 'ico' => 'image/x-icon', 'woff' => 'font/woff', 'woff2' => 'font/woff2', 'ttf' => 'font/ttf', + 'otf' => 'font/otf', 'eot' => 'application/vnd.ms-fontobject', + ]; + $file = realpath($release . '/public' . $path); + $type = $types[strtolower(pathinfo((string)$file, PATHINFO_EXTENSION))] ?? null; + // realpath() resolves the publishing symlinks; anything outside the release is refused + if ($file === false || $type === null || !is_file($file) || !str_starts_with($file, realpath($release) . '/')) { + http_response_code(404); + exit; + } + header('Content-Type: ' . $type); + header('Content-Length: ' . filesize($file)); + header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($file)) . ' GMT'); + // Content-hashed build files never change under their name (as in the .htaccess for static files). + // The Expires header keeps mod_expires from adding a second, conflicting max-age. + $maxAge = preg_match('/\\.[0-9a-f]{8}\\.\\w+$/', $file) ? 31536000 : 86400; + header('Cache-Control: public, max-age=' . $maxAge . ($maxAge === 31536000 ? ', immutable' : '')); + header('Expires: ' . gmdate('D, d M Y H:i:s', time() + $maxAge) . ' GMT'); + readfile($file); + exit; +} + +require $release . '/public/index.php'; +""" + + +def cms_api_key(): + php = '$c = include "%s/public/typo3conf/LocalConfiguration.php"; echo $c["EXTENSIONS"]["ep_products"]["cmsApiKey"] ?? "";' % APP_ROOT + key = subprocess.run(['php', '-r', php], capture_output=True, text=True, check=True).stdout.strip() + if not API_CODE_RE.match(key): + sys.exit('cmsApiKey missing or with characters the .htaccess rules cannot match literally') + return key + + +class ApiExporter: + """Snapshot /api/hotel and /api/product as JSON files, answered by TYPO3 itself. + + TYPO3 data is frozen while nobody edits, so the responses are too. A product+hotel response is + the product response with "hotel": appended (ApiMiddleware json-encodes one + array), so pair files are assembled from the single responses and a sample is checked against + TYPO3. Pairs that were not generated get the response TYPO3 gives for an unknown hotel. + """ + + def __init__(self, exp): + self.exp = exp + self.key = cms_api_key() + self.stats = {'hotels': 0, 'products': 0, 'pairs': 0, 'skipped_codes': []} + + def get(self, query): + url = self.exp.dev + '/api/' + query + for _ in range(3): # the first call of a record can be slow while TYPO3 renders its images + status, body, _ = self.exp.fetch(url, {'X-Api-Key': self.key}, log_errors=False) + if status: + return status, body + self.exp.error('api: %s failed 3 times' % query) + return status, body + + def responses(self, endpoint, codes): + """code -> body for every code TYPO3 answers with 200.""" + codes = sorted(set(codes)) + fetch = lambda c: self.get('%s?%s=%s' % (endpoint, endpoint, urllib.parse.quote(c))) + with concurrent.futures.ThreadPoolExecutor(self.exp.workers) as pool: + # Warm-up pass: responses change while TYPO3 is still generating resized images + list(pool.map(fetch, codes)) + results = pool.map(fetch, codes) + return {c: body for c, (status, body) in zip(codes, results) if status == 200} + + def write(self, path, body): + self.exp.write('/_api/' + path, self.exp.to_prod(body)) + + def run(self): + hotel_codes = collections.defaultdict(set) + for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_hotel WHERE deleted=0 AND code<>''"): + hotel_codes[uid].add(code) + aliases = mysql_rows("SELECT entity, code FROM tx_epproducts_domain_model_matchcode WHERE deleted=0 AND code<>''") + for uid, code in aliases: + hotel_codes[uid].add(code) + product_codes = collections.defaultdict(set) + for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_product WHERE deleted=0 AND code<>''"): + product_codes[uid].add(code) + pairs = mysql_rows( + "SELECT uid_local, uid_foreign FROM tx_epproducts_product_hotel_mm" + " UNION SELECT product, hotel FROM tx_epproducts_domain_model_date WHERE deleted=0" + " UNION SELECT product, hotel FROM tx_epproducts_domain_model_hotel_mapping WHERE deleted=0 AND hidden=0") + + def safe(codes): + ok = {c for c in codes if API_CODE_RE.match(c)} + self.stats['skipped_codes'] += sorted(set(codes) - ok) + return ok + + # ApiMiddleware matches matchcode aliases for products too (join on matchcode.entity) + hotels = self.responses('hotel', safe({c for cs in hotel_codes.values() for c in cs})) + products = self.responses('product', safe({c for cs in product_codes.values() for c in cs} | {c for _, c in aliases})) + # An empty code is not "missing" for ApiMiddleware: it matches a record with an empty code + for endpoint, found in (('hotel', hotels), ('product', products)): + status, body = self.get('%s?%s=' % (endpoint, endpoint)) + if status == 200: + found['@empty'] = body + for code, body in hotels.items(): + self.write('hotel/%s.json' % code, body) + for code, body in products.items(): + self.write('product/%s.json' % code, body) + self.write('product/%s/_unknown-hotel.json' % code, body[:-1] + ',"hotel":[]}') + # Product response without its closing brace, for pairs assembled by SSI (pair.shtml) + self.write('product-part/%s.part' % code, body[:-1]) + self.exp.write('/_api/pair.shtml', '' + ',"hotel":}') + written = set() + for product_uid, hotel_uid in pairs: + for pc in product_codes.get(product_uid, ()): + for hc in hotel_codes.get(hotel_uid, ()): + if pc in products and hc in hotels and (pc, hc) not in written: + written.add((pc, hc)) + self.write('product/%s/%s.json' % (pc, hc), products[pc][:-1] + ',"hotel":' + hotels[hc] + '}') + self.stats.update(hotels=len(hotels), products=len(products), pairs=len(written)) + + # The assembled files must be byte-identical to what TYPO3 answers + sample = sorted(written)[::max(1, len(written) // 40)] + [(pc, 'NO-SUCH-HOTEL') for pc in sorted(p for p in products if p != '@empty')[:5]] + for pc, hc in sample: + status, body = self.get('product?product=%s&hotel=%s' % (pc, urllib.parse.quote(hc))) + expected = products[pc][:-1] + (',"hotel":' + hotels[hc] if hc in hotels else ',"hotel":[]') + '}' + if status != 200 or body != expected: + self.exp.error('api: assembled /api/product?product=%s&hotel=%s differs from TYPO3' % (pc, hc)) + + # Error bodies exactly as ApiMiddleware sends them + for name, query in (('hotel-400', 'hotel'), ('hotel-404', 'hotel?hotel=NO-SUCH-CODE'), + ('product-400', 'product'), ('product-404', 'product?product=NO-SUCH-CODE')): + self.write('error/%s.json' % name, self.get(query)[1]) + self.write('error/403.json', self.exp.fetch(self.exp.dev + '/api/hotel?hotel=x')[1]) + return API_RULES.replace('@KEY@', self.key) + + +API_HEADER = r'''# TYPO3 backchannel API: error bodies per endpoint, as ApiMiddleware sends them +AddCharset utf-8 .json + + ErrorDocument 400 /_api/error/hotel-400.json + ErrorDocument 403 /_api/error/403.json + ErrorDocument 404 /_api/error/hotel-404.json + + + ErrorDocument 400 /_api/error/product-400.json + ErrorDocument 403 /_api/error/403.json + ErrorDocument 404 /_api/error/product-404.json + +# Product + hotel pairs that were not pre-generated are assembled by SSI from the single responses + + + Options +Includes + SetOutputFilter INCLUDES + ForceType "application/json; charset=utf-8" + + + +''' + +API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JSON snapshots taken from + # TYPO3 at export time; same status codes and bodies as ApiMiddleware + RewriteCond %{ENV:REDIRECT_STATUS} ^$ + RewriteCond %{IS_SUBREQ} false + RewriteRule ^_api/ - [R=404,L] + RewriteCond %{HTTP:X-Api-Key} !=@KEY@ + RewriteCond %{QUERY_STRING} !(^|&)key=@KEY@(&|$) + RewriteRule ^api/(hotel|product)$ - [R=403,L] + # Only file-safe codes are taken over; anything else leaves the variable unset (404 / "hotel": []) + RewriteCond %{QUERY_STRING} (^|&)hotel=([A-Za-z0-9_-]+)(&|$) + RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:%2] + RewriteCond %{QUERY_STRING} (^|&)product=([A-Za-z0-9_-]+)(&|$) + RewriteRule ^api/product$ - [E=API_PRODUCT:%2] + # An empty code matches a record with an empty code in TYPO3: snapshot file "@empty" (never a real code) + RewriteCond %{QUERY_STRING} (^|&)hotel=(&|$) + RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:@empty] + RewriteCond %{QUERY_STRING} (^|&)product=(&|$) + RewriteRule ^api/product$ - [E=API_PRODUCT:@empty] + RewriteCond %{QUERY_STRING} !(^|&)hotel= + RewriteRule ^api/hotel$ - [R=400,L] + RewriteCond %{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)$ + RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f + RewriteRule ^api/hotel$ _api/hotel/%{ENV:API_HOTEL}.json [L] + RewriteRule ^api/hotel$ - [R=404,L] + RewriteCond %{QUERY_STRING} !(^|&)product= + RewriteRule ^api/product$ - [R=400,L] + RewriteCond %{QUERY_STRING} (^|&)hotel= + RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$ + RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json -f + RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json [L] + # Pair not pre-generated but both known: assembled by SSI (exactly what TYPO3 returns) + + RewriteCond %{QUERY_STRING} (^|&)hotel= + RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$ + RewriteCond %{DOCUMENT_ROOT}/_api/product-part/%{ENV:API_PRODUCT}.part -f + RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f + RewriteRule ^api/product$ _api/pair.shtml [L] + + # Hotel unknown (or no mod_include): TYPO3 answers with "hotel": [] + RewriteCond %{QUERY_STRING} (^|&)hotel= + RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$ + RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json -f + RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json [L] + RewriteCond %{QUERY_STRING} !(^|&)hotel= + RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$ + RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}.json -f + RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}.json [L] + RewriteRule ^api/product$ - [R=404,L] +''' + +REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n' + +STATIC_REWRITE = r''' + # Static export: reject multipart requests (CVE-2026-109618 / -109619) + RewriteCond %{HTTP:Content-Type} multipart [NC] + RewriteRule ^ - [F,L] + # No PHP is ever served, also not via a symlinked fileadmin + # (/buchung/ is exempt: it redirects to the booking system further down) + RewriteCond %{REQUEST_URI} !^/buchung/ + RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC] + # Search does not work statically: any search page goes to the homepage + RewriteRule (^|/)suche/?$ / [R=302,L] +''' + + +HYBRID_HEADER = '''### Begin: Static export (hybrid) ### +# Generated by .ddev/static-export/export.py --hybrid from public/.htaccess. +# Files of the static export win; anything else goes to TYPO3 13 through index.php. +DirectoryIndex index.html +ErrorDocument 404 /404.html +Options -Indexes -MultiViews +### End: Static export (hybrid) ### + +''' + +HYBRID_REWRITE = r''' + # Reject multipart requests (CVE-2026-109618 / -109619); the search sends URL-encoded forms + RewriteCond %{HTTP:Content-Type} multipart [NC] + RewriteRule ^ - [F,L] + # No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below + # (/buchung/ is exempt: it redirects to the booking system further down) + RewriteCond %{REQUEST_URI} !^/buchung/ + RewriteCond %{ENV:REDIRECT_STATUS} ^$ [OR] + RewriteCond %{REQUEST_URI} !^/index\.php$ + RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC] + # TYPO3 13 backend and install tool stay closed to the public + RewriteRule ^typo3(/|$) - [F,L] +''' + +HYBRID_CATCH_ALL = ''' # Not part of the static export (a directory without index.html counts as missing) => TYPO3 13 + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME}/index.html !-f + RewriteRule ^.*$ %{ENV:CWD}index.php [QSA,L] +''' + + +def build_htaccess(hybrid=False): + """TYPO3's .htaccess without the parts that hand requests to PHP (hybrid: with the hand-over).""" + with open(os.path.join(PUBLIC, '.htaccess'), encoding='utf-8') as fh: + text = fh.read() + text, sfc = re.subn(r'[ \t]*### Begin: StaticFileCache \(preparation\).*?### End: StaticFileCache ###\n', '', text, flags=re.S) + text, catch_all = re.subn(r'(?:[ \t]*#[^\n]*\n)*(?:[ \t]*RewriteCond %\{REQUEST_FILENAME\} !-[fdl]\n){3}[ \t]*RewriteRule \^\.\*\$ %\{ENV:CWD\}index\.php \[QSA,L\]\n', + lambda m: HYBRID_CATCH_ALL if hybrid else '', text) + rewrite = HYBRID_REWRITE if hybrid else STATIC_REWRITE + text, engine = re.subn(r'(\n[ \t]*RewriteEngine On\n)', r'\1' + (rewrite + REDIRECTS_PLACEHOLDER).replace('\\', '\\\\'), text, count=1) + leftover = re.sub(r'#[^\n]*', '', text.replace(HYBRID_CATCH_ALL, '') if hybrid else text) + if (sfc, catch_all, engine) != (1, 1, 1) or re.search(r'RewriteRule[^\n]*index\.php', leftover): + sys.exit('public/.htaccess changed shape, adjust build_htaccess() (sfc=%d, catch-all=%d, engine=%d)' % (sfc, catch_all, engine)) + return (HYBRID_HEADER if hybrid else STATIC_HEADER) + text + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--site', action='append', help='site identifier from config/sites (repeatable, default: all)') + ap.add_argument('--url', action='append', help='export only these URLs (dev URLs) instead of the sitemap') + ap.add_argument('--no-follow', action='store_true', help='do not follow internal links beyond the start URLs') + ap.add_argument('--out', default=os.path.join(APP_ROOT, 'var', 'static-export')) + ap.add_argument('--workers', type=int, default=8) + ap.add_argument('--hybrid', action='store_true', help='leave product pages and search to TYPO3 13 (see module doc)') + ap.add_argument('--typo3-current', help='hybrid: absolute path of the TYPO3 13 "current" release symlink on the server') + ap.add_argument('--live-ajax', action='store_true', help='hybrid: keep AJAX URLs that TYPO3 13 serves instead of pre-fetching them') + args = ap.parse_args() + if args.hybrid and not args.typo3_current: + ap.error('--hybrid needs --typo3-current') + sys.stdout.reconfigure(line_buffering=True) + + all_sites = load_sites() + selected = {k: v for k, v in all_sites.items() if not args.site or k in args.site} + if not selected: + sys.exit('No matching site in %s' % SITES_DIR) + + htaccess = build_htaccess(args.hybrid) + failed = False + for name, site in selected.items(): + out = os.path.join(args.out, name) + shutil.rmtree(out, ignore_errors=True) + exp = Exporter(name, site, all_sites, out, follow=not args.no_follow and not args.url, workers=args.workers, + hybrid=args.hybrid, live_ajax=args.live_ajax) + print('== %s: %s → %s' % (name, site['dev'], site['prod'])) + if args.url: + start = [u for u in args.url if u.startswith(site['dev'])] + else: + start = [site['dev'] + '/'] + exp.sitemap() + if not args.hybrid: + # Linked from nowhere (handed out to groups and customers); in hybrid mode TYPO3 13 serves them + routes = route_urls(site) + start += routes + if routes: + print(' route URLs (/gruppe/, /reiseinformationen/): %d' % len(routes)) + exp.special_files() + exp.crawl(start) + rules = exp.redirects() + site_htaccess = htaccess + if args.hybrid: + # TYPO3 13 behind index.php + # /_assets/ is served by the stub too (no symlink: not every vhost follows one) + exp.write('/index.php', FRONT_CONTROLLER_STUB % args.typo3_current.rstrip('/')) + if name in API_SITES and not args.url and not args.hybrid: + api = ApiExporter(exp) + rules = api.run() + rules + site_htaccess = API_HEADER + htaccess + print(' api: %d hotels, %d products, %d product/hotel pairs, codes skipped (not file-safe): %s' + % (api.stats['hotels'], api.stats['products'], api.stats['pairs'], ', '.join(api.stats['skipped_codes']) or '-')) + if exp.crawl_redirects: + rules += '\t# Redirects returned by TYPO3 while crawling\n' + '\n'.join(sorted(exp.crawl_redirects)) + '\n' + exp.write('/.htaccess', site_htaccess.replace(REDIRECTS_PLACEHOLDER, '\n' + rules)) + copied, missing = exp.copy_assets() + print(' pages: %d, ajax tables: %d, redirects: %d (%d skipped, see %s-redirects-skipped.txt), assets: %d copied, %d missing' + % (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing))) + print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search'])) + if args.hybrid: + print(' left to TYPO3 13: %d pages' % exp.stats['dynamic']) + for path in sorted(missing)[:20]: + print(' missing asset: %s' % path) + for msg in exp.errors[:20]: + print(' error: %s' % msg) + if len(exp.errors) > 20: + print(' ... %d more errors' % (len(exp.errors) - 20)) + failed = failed or bool(exp.errors) + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main()