Files
website-typo3/.ddev/static-export/export.py
T

937 lines
46 KiB
Python

#!/usr/bin/env python3
"""Static export of the TYPO3 sites running in DDEV.
Crawls each site (sitemap + internal links), pre-fetches the AJAX dates and
price tables of product pages, rewrites *.ddev.site hosts to the production
domains and copies every referenced static asset. The result is a set of
plain document roots that run without PHP.
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
and the search are left out (TYPO3 13 serves them through index.php), the search
UI stays (except the search bar), and the API is answered live.
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
[--hybrid --typo3-current PATH [--live-ajax]]
"""
import argparse
import collections
import concurrent.futures
import hashlib
import html
import http.client
import os
import re
import shutil
import subprocess
import sys
import threading
import urllib.error
import urllib.parse
import urllib.request
APP_ROOT = os.environ.get('DDEV_APPROOT', '/var/www/html')
PUBLIC = os.path.join(APP_ROOT, 'public')
SITES_DIR = os.path.join(APP_ROOT, 'config', 'sites')
# Hosts that pull from the production origin; their paths must exist in the export
CDN_HOSTS = {'cdn.ep-reisen.de'}
# Any attribute pointing at TYPO3's AJAX page type (product tables, HTMX teaser
# popups, ...) is pre-fetched; JS may append "&pa=..." to these URLs
AJAX_PATH = '/ajax/'
# Forms posting to these TYPO3 plugins (contact, enquiry, application, booking, newsletter)
# cannot work statically and are replaced by a contact notice
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
# and links to the search page (menu icon, "Alle anzeigen" with preset filters)
SEARCH_ELEMENTS = (
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
('div', re.compile(r'<div\b[^>]*\bdata-controller="mobile-search"[^>]*>')),
('button', re.compile(r'<button\b[^>]*\bdata-overlay-toggle-id-value="(?:mobile-)?search"[^>]*>')),
('a', re.compile(r'<a\b[^>]*\bhref="[^"]*/suche/(?:\?[^"]*)?"[^>]*>')),
)
SEARCH_PAGE_RE = re.compile(r'/suche/$')
# Hybrid: only the search bar goes (TYPO3 13 has no search bar plugin); links, menu icons and
# the mobile filter overlay stay, the search itself is served by TYPO3 13
HYBRID_SEARCH_ELEMENTS = (
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
('button', re.compile(r'<button\b[^>]*\bdata-controller="searchbar-toggle"[^>]*>')),
)
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
# (404 on TYPO3 13 for popups pointing at another site)
LIVE_AJAX = {
('AjaxTable', 'datestable'), ('AjaxTable', 'pricetable'), ('AjaxTable', 'pricetableHtml'),
('AjaxTable', 'eventPricetable'),
('AjaxCalendar', 'priceTable'), ('AjaxCalendar', 'priceCalendar'),
('AjaxSearch', 'searchresult'), ('AjaxFilterPanel', 'index'), ('AjaxWatchlist', 'list'),
}
LIVE_AJAX_PARAM_RE = re.compile(r'tx_epproducts_ajax(?:%5B|\[)(controller|action)(?:%5D|\])=(\w+)')
# Contact shown instead of forms: display phone, tel: number, e-mail (as in each site's footer)
CONTACTS = {
'ep-reisen': ('0221 272276-0', '+492212722760', '[email protected]'),
'ski-boarderweek': ('0221 272276-47', '+4922127227647', '[email protected]'),
'snowzone': ('0221 272276-47', '+4922127227647', '[email protected]'),
'unichamp': ('0221 272276-47', '+4922127227647', '[email protected]'),
'ser': ('0221 272276-41', '+4922127227641', '[email protected]'),
'alvac': ('0221 272276-30', '+4922127227630', '[email protected]'),
'cllt': ('0221 272276-30', '+4922127227630', '[email protected]'),
}
DEFAULT_CONTACT = CONTACTS['ep-reisen']
CONTACT_TEXT = {
'de': ('Dieses Formular ist vorübergehend nicht verfügbar.',
'Du hast Fragen oder möchtest buchen? Unser Kundenservice hilft dir gerne weiter:'),
'en': ('This form is temporarily unavailable.',
'Questions or want to book? Our customer service is happy to help:'),
}
CONTACT_NOTICE = '''<div class="static-contact-notice" style="background:#ebf3fa;border-left:4px solid #3d8ccb;padding:24px;margin:16px 0">
<p style="margin:0 0 8px;font-weight:700">{lead}</p>
<p style="margin:0 0 16px">{text}</p>
<p style="margin:0;display:flex;flex-wrap:wrap;gap:8px">
<a href="tel:{tel}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{phone}</a>
<a href="mailto:{email}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{email}</a>
</p>
</div>'''
def replace_elements(text, tag, start_re, replacement):
"""Replace each element whose start tag matches, including its balanced content."""
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
out, pos, count = [], 0, 0
for m in start_re.finditer(text):
if m.start() < pos:
continue # nested inside an element already replaced
depth, end = 1, None
for t in token_re.finditer(text, m.end()):
depth += -1 if t.group(1) else 1
if depth == 0:
end = t.end()
break
if end is None:
continue # unbalanced markup, leave it alone
out += [text[pos:m.start()], replacement]
pos, count = end, count + 1
out.append(text[pos:])
return ''.join(out), count
URL_ATTR_RE = re.compile(r'(?:href|src|srcset|data-src|data-srcset|content|poster|action)="([^"]*)"')
CSS_URL_RE = re.compile(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)')
ASSET_EXT_RE = re.compile(r'\.(css|js|mjs|json|map|svg|png|jpe?g|gif|webp|avif|ico|woff2?|ttf|eot|otf|pdf|mp4|webm|txt|xml|webmanifest)$', re.I)
class NoRedirect(urllib.request.HTTPRedirectHandler):
"""Report redirects instead of following them, so they become rewrite rules."""
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
OPENER = urllib.request.build_opener(NoRedirect)
def rewrite_rule(path, location, status):
pattern = re.escape(urllib.parse.unquote(path).strip('/')).replace('\\ ', '\\x20')
target = re.sub(r'([$%])', r'\\\1', location)
return '\tRewriteRule "^%s/?$" "%s" [R=%d,L,NE]' % (pattern, target, status)
def load_sites(only=None):
"""Read base and Development base variant from each site config."""
sites = {}
for name in sorted(os.listdir(SITES_DIR)):
path = os.path.join(SITES_DIR, name, 'config.yaml')
if not os.path.isfile(path) or (only and name not in only):
continue
with open(path, encoding='utf-8') as fh:
text = fh.read()
prod = re.search(r"^base:\s*'?([^'\s]+)", text, re.M)
dev = re.search(r"base:\s*'?(https://[^'\s]+\.ddev\.site)'?\s*\n\s*condition:\s*'applicationContext == \"Development\"'", text)
if prod and dev and prod.group(1).startswith('http'):
root = re.search(r'^rootPageId:\s*(\d+)', text, re.M)
sites[name] = {'prod': prod.group(1).rstrip('/'), 'dev': dev.group(1).rstrip('/'),
'root': int(root.group(1)) if root else 0, 'languages': site_languages(text)}
return sites
def site_languages(text):
"""languageId => base path prefix ('' for '/', '/en' for '/en/') of a site config."""
section = re.search(r'^languages:\n((?:[ \t-].*\n|\n)*)', text, re.M)
languages = {}
for item in re.split(r'^ -[ \t]*\n?', section.group(1) if section else '', flags=re.M):
language_id = re.search(r"^\s*languageId:\s*'?(\d+)", item, re.M)
base = re.search(r"^\s{4}base:\s*'?([^'\s]+)", item, re.M)
if language_id and base and base.group(1).startswith('/'):
languages[int(language_id.group(1))] = base.group(1).rstrip('/')
return languages
class Exporter:
def __init__(self, name, site, all_sites, out_dir, follow, workers, hybrid=False, live_ajax=False):
self.site = site
self.hybrid = hybrid
self.live_ajax = live_ajax
self.dynamic = dynamic_paths(site) if hybrid else set()
self.route_prefixes = set(route_pages(site).values()) if hybrid else set()
self.contact = CONTACTS.get(name, DEFAULT_CONTACT)
self.dev = site['dev']
self.dev_host = urllib.parse.urlsplit(self.dev).netloc
# Editors often paste absolute production URLs; those are internal links too
self.prod_host = urllib.parse.urlsplit(site['prod']).netloc
self.ajax_results = {}
self.ajax_locks = {}
self.out = out_dir
self.follow = follow
self.workers = workers
# dev → prod for every site, so cross-site links come out right too
self.host_map = [(s['dev'], s['prod']) for s in all_sites.values()]
# Some pages get their AJAX URLs below a parent slug (/reisen-fuer-gruppen/ajax/html?...)
self.ajax_attr_re = re.compile(r'([a-zA-Z][\w:.-]*)="((?:%s)?(?:/[\w-]+)*%s[^"]*)"' % (re.escape(self.dev), re.escape(AJAX_PATH)))
self.seen = set()
self.assets = set()
self.lock = threading.Lock()
self.errors = []
self.crawl_redirects = []
self.follow_redirects = follow
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'search': 0, 'dynamic': 0}
# --- helpers -----------------------------------------------------------
def fetch(self, url, extra_headers=None, log_errors=True):
"""Return status, body and (for redirects) the Location header."""
headers = {'User-Agent': 'ep-static-export'}
headers.update(extra_headers or {})
parts = urllib.parse.urlsplit(url)
if parts.scheme == 'https' and parts.hostname.endswith('.ddev.site'):
# Talk to the container's own web server, bypassing ddev-router (works for hostnames
# added without "ddev restart"); TYPO3 still sees HTTPS via the forwarded header
url = urllib.parse.urlunsplit(('http', '127.0.0.1') + tuple(parts[2:]))
headers.update({'Host': parts.hostname, 'X-Forwarded-Proto': 'https'})
req = urllib.request.Request(url, headers=headers)
try:
with OPENER.open(req, timeout=60) as res:
return res.status, res.read().decode('utf-8', 'replace'), None
except urllib.error.HTTPError as e:
return e.code, e.read().decode('utf-8', 'replace'), e.headers.get('Location')
except Exception as e: # noqa: BLE001 - report and continue
if log_errors:
self.error('%s: %s' % (url, e))
return 0, '', None
def error(self, msg):
with self.lock:
self.errors.append(msg)
def write(self, rel_path, content):
path = os.path.join(self.out, rel_path.lstrip('/'))
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w', encoding='utf-8') as fh:
fh.write(content)
def to_prod(self, text):
for dev, prod in self.host_map:
text = text.replace(dev, prod)
text = text.replace(dev.replace('/', '\\/'), prod.replace('/', '\\/'))
text = text.replace(urllib.parse.quote(dev, safe=''), urllib.parse.quote(prod, safe=''))
return text
def local_path(self, url):
"""Path on this site for a URL, or None if it points elsewhere."""
parts = urllib.parse.urlsplit(html.unescape(url))
if parts.scheme and parts.scheme not in ('http', 'https'):
return None
if parts.netloc and parts.netloc not in (self.dev_host, self.prod_host) and parts.netloc not in CDN_HOSTS:
return None
if not parts.path.startswith('/') or parts.path.startswith('//'):
return None
return parts
def collect(self, text):
"""Collect asset paths and (if following) internal page links."""
links = set()
for raw in URL_ATTR_RE.findall(text) + CSS_URL_RE.findall(text):
for candidate in raw.split(','): # srcset
candidate = candidate.strip().split(' ')[0]
parts = self.local_path(candidate)
if not parts:
continue
path = urllib.parse.unquote(parts.path)
if ASSET_EXT_RE.search(path):
self.assets.add(path)
elif self.follow and not parts.query and parts.netloc in ('', self.dev_host, self.prod_host) \
and not path.startswith(('/typo3', AJAX_PATH)):
links.add(self.dev + path)
return links
# --- pages and AJAX tables ---------------------------------------------
def ajax_file(self, url):
"""Fetch an AJAX table, store it, return its static URL."""
url = html.unescape(url)
if url.startswith('/'):
url = self.dev + url
name = '/_ajax/%s.html' % hashlib.sha1(url.replace(self.dev, '').encode()).hexdigest()
with self.lock:
name_lock = self.ajax_locks.setdefault(name, threading.RLock())
# Every reference waits for the first fetch, so a failed table keeps its original URL everywhere
with name_lock:
if name not in self.ajax_results:
# Set before fetching: a response referencing itself must not recurse.
# The JS appends "&pa=..."; the dummy query keeps that harmless
self.ajax_results[name] = name + '?s=1'
status, body, _ = self.fetch(url)
if status != 200:
self.error('%s → %s' % (url, status))
self.ajax_results[name] = url
else:
self.write(name, self.to_prod(self.process(self.strip_dynamic(body, ''))))
with self.lock:
self.stats['ajax'] += 1
return self.ajax_results[name]
def strip_dynamic(self, text, path):
"""Swap contact forms for a contact notice and drop the search UI."""
phone, tel, email = self.contact
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
removed = 0
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
text, n = replace_elements(text, tag, start_re, '')
removed += n
with self.lock:
self.stats['forms'] += forms
self.stats['search'] += removed
return text
def process(self, text):
"""Replace AJAX table URLs (recursively fetching them) and collect assets."""
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
'%s="%s"' % (m.group(1), html.escape(self.ajax_file(m.group(2)))), text)
self.collect(text)
return text
def is_live_ajax(self, url):
"""Hybrid --live-ajax: keep the URL when TYPO3 13 serves the action."""
params = dict(LIVE_AJAX_PARAM_RE.findall(html.unescape(url)))
return self.live_ajax and (params.get('controller'), params.get('action')) in LIVE_AJAX
def is_dynamic(self, path):
"""Hybrid: product pages (with their /detail/ variants) and search pages belong to TYPO3 13."""
path = path.rstrip('/') + '/'
return (path in self.dynamic or any(path.startswith(p + 'detail/') for p in self.dynamic)
or any(path.startswith(p) and path != p for p in self.route_prefixes))
def safe_page(self, url):
try:
return self.page(url)
except Exception as e: # noqa: BLE001 - one broken page must not stop the export
self.error('%s: %s: %s' % (url, type(e).__name__, e))
return set()
def page(self, url):
status, body, location = self.fetch(url)
if status in (301, 302, 303, 307, 308) and location:
location = urllib.parse.urljoin(url, location)
with self.lock:
self.crawl_redirects.append(rewrite_rule(urllib.parse.urlsplit(url).path, self.to_prod(location), status))
internal = location.startswith(self.dev + '/') and '?' not in location
return {location} if internal and self.follow_redirects else set()
if status != 200:
self.error('%s → %s' % (url, status))
return set()
path = urllib.parse.urlsplit(url).path
if self.hybrid and self.is_dynamic(path):
# Served by TYPO3 13 through index.php; only its links are followed
with self.lock:
self.stats['dynamic'] += 1
return self.collect(body)
if not self.hybrid and SEARCH_PAGE_RE.search(path):
# Search does not work statically: send visitors to the homepage instead
with self.lock:
self.crawl_redirects.append(rewrite_rule(path, self.site['prod'] + '/', 302))
return set()
# Extensionless paths become directories, so /a and /a/b/ can both exist
target = path if re.search(r'\.[a-z0-9]+$', path, re.I) else path.rstrip('/') + '/index.html'
body = self.strip_dynamic(body, path)
links = self.collect(body)
self.write(target, self.to_prod(self.process(body)))
with self.lock:
self.stats['pages'] += 1
return links
def sitemap(self):
"""Export sitemap index and children as plain files, return page URLs."""
status, index, _ = self.fetch(self.dev + '/sitemap.xml')
if status != 200:
print(' note: no sitemap.xml (%s), crawling from / only' % status)
return []
urls = []
for loc in re.findall(r'<loc>([^<]+)</loc>', index):
child_url = html.unescape(loc)
name = urllib.parse.parse_qs(urllib.parse.urlsplit(child_url).query).get('sitemap', ['pages'])[0]
_, child, _ = self.fetch(child_url)
urls += [html.unescape(u) for u in re.findall(r'<loc>([^<]+)</loc>', child)]
self.write('/sitemap-%s.xml' % name, self.to_prod(child))
index = index.replace(loc, self.dev + '/sitemap-%s.xml' % name)
self.write('/sitemap.xml', self.to_prod(index))
return [u for u in urls if u.startswith(self.dev)]
def crawl(self, start):
queue = list(dict.fromkeys(start))
self.seen.update(queue)
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
while queue:
batch, queue = queue, []
for links in pool.map(self.safe_page, batch):
for link in links:
if link not in self.seen:
self.seen.add(link)
queue.append(link)
def redirects(self):
"""Backend redirects (sys_redirect) as RewriteRules, targets resolved by TYPO3 itself."""
host = urllib.parse.urlsplit(self.site['prod']).netloc
where = "deleted=0 AND disabled=0 AND source_host IN ('%s', '*')" % host
try:
out = subprocess.run(
['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e',
'SELECT source_path FROM sys_redirect WHERE %s AND is_regexp=0; '
'SELECT COUNT(*) FROM sys_redirect WHERE %s AND is_regexp=1' % (where, where)],
capture_output=True, text=True, check=True).stdout.splitlines()
except (OSError, subprocess.CalledProcessError) as e:
self.error('sys_redirect: %s' % e)
return ''
paths, regexp = sorted(set(out[:-1])), int(out[-1])
skipped = ['regexp redirects (not exported): %d' % regexp] if regexp else []
def resolve(path):
if '?' in path:
return path, 0, 'query string in source path'
conn = http.client.HTTPConnection('127.0.0.1', 80, timeout=60)
try:
conn.request('GET', urllib.parse.quote(path, safe="/%:@!$&'()*+,;=-._~"), headers={'Host': host})
res = conn.getresponse()
res.read()
return path, res.status, res.getheader('Location')
finally:
conn.close()
rules = ['\t# Redirects from the TYPO3 backend (sys_redirect), resolved by TYPO3 at export time']
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
for path, status, location in pool.map(resolve, paths):
if status not in (301, 302, 303, 307, 308) or not location:
# Mostly dead entries (target page gone) that 404 in production already
skipped.append('%s → %s %s' % (path, status, location or ''))
continue
# Resolve like a browser: TYPO3 sends "https:/path" for relative targets
location = urllib.parse.urljoin(self.dev + path, location)
if location.rstrip('/') == (self.dev + path).rstrip('/'):
# TYPO3 matches "/a" for "/a/" too, so a redirect to the page at "/a/" loops
skipped.append('%s → %s (redirects to itself)' % (path, location))
continue
target = urllib.parse.urlsplit(location)
if target.netloc == self.dev_host and ASSET_EXT_RE.search(target.path):
self.assets.add(urllib.parse.unquote(target.path))
rules.append(rewrite_rule(path, self.to_prod(location), status))
self.stats['redirects'] = len(rules) - 1
self.stats['redirects_skipped'] = len(skipped)
with open(self.out.rstrip('/') + '-redirects-skipped.txt', 'w', encoding='utf-8') as fh:
fh.write('\n'.join(skipped) + '\n')
return '\n'.join(rules) + '\n'
def special_files(self):
status, body, _ = self.fetch(self.dev + '/robots.txt')
if status == 200:
self.write('/robots.txt', self.to_prod(body))
# TYPO3 error page, served by Apache via ErrorDocument 404
status, body, _ = self.fetch(self.dev + '/static-export-not-found/')
if status == 404 and body:
self.write('/404.html', self.to_prod(self.process(self.strip_dynamic(body, '/'))))
# --- static assets -----------------------------------------------------
def copy_assets(self):
pending, copied, missing = list(self.assets), set(), []
while pending:
path = pending.pop()
if path in copied:
continue
copied.add(path)
src = os.path.join(PUBLIC, path.lstrip('/'))
if not os.path.isfile(src):
missing.append(path)
continue
dst = os.path.join(self.out, path.lstrip('/'))
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copy2(src, dst)
if path.endswith('.css'): # fonts and images referenced from CSS
with open(src, encoding='utf-8', errors='replace') as fh:
for ref in CSS_URL_RE.findall(fh.read()):
if ref.startswith(('data:', 'http', '#')):
continue
ref = urllib.parse.urlsplit(ref).path
pending.append(os.path.normpath(os.path.join(os.path.dirname(path), ref)))
return len(copied) - len(missing), missing
STATIC_HEADER = '''### Begin: Static export ###
# Generated by .ddev/static-export/export.py from public/.htaccess.
# No PHP: TYPO3's index.php catch-all and StaticFileCache rules are removed.
DirectoryIndex index.html
ErrorDocument 404 /404.html
Options -Indexes -MultiViews
<IfModule mod_php7.c>
php_flag engine off
</IfModule>
### End: Static export ###
'''
# --- TYPO3 backchannel API (/api/hotel, /api/product) ----------------------------------------
# MyE&P calls the API on this site; ApiMiddleware itself answers on every host
API_SITES = {'ep-reisen'}
API_CODE_RE = re.compile(r'^[A-Za-z0-9_-]+$')
def mysql_rows(sql):
out = subprocess.run(['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', sql],
capture_output=True, text=True, check=True).stdout
return [line.split('\t') for line in out.splitlines() if line]
# Records that only exist as URL arguments of one page (route enhancers), not linked anywhere:
# (page uid, table, field, pattern the value must match or None)
ROUTE_RECORDS = (
(1846, 'tx_epproducts_domain_model_product', 'path_segment', None), # /gruppe/{product}
(2736, 'tx_epproducts_domain_model_travelinfo', 'path_segment', None), # /reiseinformationen/{name}
(2736, 'tx_epproducts_domain_model_travelinfo', 'travel_code', r'[A-Z]+\d{6}'), # /reiseinformationen/{code}
)
def page_tree_check(site):
"""Return a function telling whether a default-language page uid belongs to the site."""
parents = {int(uid): int(pid) for uid, pid in mysql_rows('SELECT uid, pid FROM pages WHERE sys_language_uid = 0')}
def in_site(uid):
for _ in range(100):
if uid == site['root']:
return True
if uid not in parents:
return False
uid = parents[uid]
return False
return in_site
def route_pages(site):
"""Slug (with trailing slash) of each route page of ROUTE_RECORDS on this site, by page uid."""
in_site = page_tree_check(site)
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
pages = {}
for uid in sorted({r[0] for r in ROUTE_RECORDS}):
rows = mysql_rows('SELECT slug FROM pages WHERE uid = %d AND deleted = 0 AND hidden = 0' % uid)
if rows and in_site(uid):
pages[uid] = base + rows[0][0].rstrip('/') + '/'
return pages
def route_urls(site):
"""Start URLs for the records of ROUTE_RECORDS (not reachable through links or the sitemap)."""
pages = route_pages(site)
urls = set()
for uid, table, field, pattern in ROUTE_RECORDS:
if uid not in pages:
continue
for (value,) in mysql_rows("SELECT DISTINCT %s FROM %s WHERE deleted = 0 AND hidden = 0 AND %s <> ''" % (field, table, field)):
if pattern is None or re.fullmatch(pattern, value):
urls.add(site['dev'].rstrip('/') + pages[uid] + urllib.parse.quote(value) + '/')
return sorted(urls)
def dynamic_paths(site):
"""Paths (with trailing slash) of the pages TYPO3 13 serves in hybrid mode: pages of this site
holding the product detail or search result plugin, in every language of the site."""
in_site = page_tree_check(site)
rows = mysql_rows(
"SELECT DISTINCT p.slug, p.sys_language_uid, IF(p.sys_language_uid = 0, p.uid, p.l10n_parent) "
"FROM pages p JOIN tt_content c ON c.pid = p.uid "
"WHERE p.deleted = 0 AND p.hidden = 0 AND c.deleted = 0 AND c.hidden = 0 AND c.CType = 'list' "
"AND c.list_type IN ('epproducts_product_detail', 'epproducts_searchresult')")
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
paths = set()
for slug, language, default_uid in rows:
if int(language) in site['languages'] and in_site(int(default_uid)):
paths.add(base + site['languages'][int(language)] + slug.rstrip('/') + '/')
return paths
FRONT_CONTROLLER_STUB = """<?php
// Generated by .ddev/static-export/export.py --hybrid: everything not in this static export is
// handled by TYPO3 13 (product pages, /gruppe/, travel info, search, /api/*, /ajax/*, 404).
// readlink() is not cached, unlike the realpath of a symlinked include path, so right after a
// deploy every request already loads the new release (under new paths, so OPcache needs no reset).
$current = %r;
$release = readlink($current);
if ($release === false) {
http_response_code(503);
exit;
}
$release = $release[0] === '/' ? $release : dirname($current) . '/' . $release;
// Public extension resources (/_assets/<hash>/...) come from the release as well: Apache does not
// follow a symlink from the docroot into the release on every vhost, PHP does.
$path = rawurldecode((string)strtok($_SERVER['REQUEST_URI'] ?? '', '?'));
if (str_starts_with($path, '/_assets/')) {
$types = [
'css' => 'text/css; charset=utf-8', 'js' => 'text/javascript; charset=utf-8', 'json' => 'application/json',
'map' => 'application/json', 'svg' => 'image/svg+xml', 'png' => 'image/png', 'jpg' => 'image/jpeg',
'jpeg' => 'image/jpeg', 'gif' => 'image/gif', 'webp' => 'image/webp', 'avif' => 'image/avif',
'ico' => 'image/x-icon', 'woff' => 'font/woff', 'woff2' => 'font/woff2', 'ttf' => 'font/ttf',
'otf' => 'font/otf', 'eot' => 'application/vnd.ms-fontobject',
];
$file = realpath($release . '/public' . $path);
$type = $types[strtolower(pathinfo((string)$file, PATHINFO_EXTENSION))] ?? null;
// realpath() resolves the publishing symlinks; anything outside the release is refused
if ($file === false || $type === null || !is_file($file) || !str_starts_with($file, realpath($release) . '/')) {
http_response_code(404);
exit;
}
header('Content-Type: ' . $type);
header('Content-Length: ' . filesize($file));
header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($file)) . ' GMT');
// Content-hashed build files never change under their name (as in the .htaccess for static files).
// The Expires header keeps mod_expires from adding a second, conflicting max-age.
$maxAge = preg_match('/\\.[0-9a-f]{8}\\.\\w+$/', $file) ? 31536000 : 86400;
header('Cache-Control: public, max-age=' . $maxAge . ($maxAge === 31536000 ? ', immutable' : ''));
header('Expires: ' . gmdate('D, d M Y H:i:s', time() + $maxAge) . ' GMT');
readfile($file);
exit;
}
require $release . '/public/index.php';
"""
def cms_api_key():
php = '$c = include "%s/public/typo3conf/LocalConfiguration.php"; echo $c["EXTENSIONS"]["ep_products"]["cmsApiKey"] ?? "";' % APP_ROOT
key = subprocess.run(['php', '-r', php], capture_output=True, text=True, check=True).stdout.strip()
if not API_CODE_RE.match(key):
sys.exit('cmsApiKey missing or with characters the .htaccess rules cannot match literally')
return key
class ApiExporter:
"""Snapshot /api/hotel and /api/product as JSON files, answered by TYPO3 itself.
TYPO3 data is frozen while nobody edits, so the responses are too. A product+hotel response is
the product response with "hotel": <hotel response> appended (ApiMiddleware json-encodes one
array), so pair files are assembled from the single responses and a sample is checked against
TYPO3. Pairs that were not generated get the response TYPO3 gives for an unknown hotel.
"""
def __init__(self, exp):
self.exp = exp
self.key = cms_api_key()
self.stats = {'hotels': 0, 'products': 0, 'pairs': 0, 'skipped_codes': []}
def get(self, query):
url = self.exp.dev + '/api/' + query
for _ in range(3): # the first call of a record can be slow while TYPO3 renders its images
status, body, _ = self.exp.fetch(url, {'X-Api-Key': self.key}, log_errors=False)
if status:
return status, body
self.exp.error('api: %s failed 3 times' % query)
return status, body
def responses(self, endpoint, codes):
"""code -> body for every code TYPO3 answers with 200."""
codes = sorted(set(codes))
fetch = lambda c: self.get('%s?%s=%s' % (endpoint, endpoint, urllib.parse.quote(c)))
with concurrent.futures.ThreadPoolExecutor(self.exp.workers) as pool:
# Warm-up pass: responses change while TYPO3 is still generating resized images
list(pool.map(fetch, codes))
results = pool.map(fetch, codes)
return {c: body for c, (status, body) in zip(codes, results) if status == 200}
def write(self, path, body):
self.exp.write('/_api/' + path, self.exp.to_prod(body))
def run(self):
hotel_codes = collections.defaultdict(set)
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_hotel WHERE deleted=0 AND code<>''"):
hotel_codes[uid].add(code)
aliases = mysql_rows("SELECT entity, code FROM tx_epproducts_domain_model_matchcode WHERE deleted=0 AND code<>''")
for uid, code in aliases:
hotel_codes[uid].add(code)
product_codes = collections.defaultdict(set)
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_product WHERE deleted=0 AND code<>''"):
product_codes[uid].add(code)
pairs = mysql_rows(
"SELECT uid_local, uid_foreign FROM tx_epproducts_product_hotel_mm"
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_date WHERE deleted=0"
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_hotel_mapping WHERE deleted=0 AND hidden=0")
def safe(codes):
ok = {c for c in codes if API_CODE_RE.match(c)}
self.stats['skipped_codes'] += sorted(set(codes) - ok)
return ok
# ApiMiddleware matches matchcode aliases for products too (join on matchcode.entity)
hotels = self.responses('hotel', safe({c for cs in hotel_codes.values() for c in cs}))
products = self.responses('product', safe({c for cs in product_codes.values() for c in cs} | {c for _, c in aliases}))
# An empty code is not "missing" for ApiMiddleware: it matches a record with an empty code
for endpoint, found in (('hotel', hotels), ('product', products)):
status, body = self.get('%s?%s=' % (endpoint, endpoint))
if status == 200:
found['@empty'] = body
for code, body in hotels.items():
self.write('hotel/%s.json' % code, body)
for code, body in products.items():
self.write('product/%s.json' % code, body)
self.write('product/%s/_unknown-hotel.json' % code, body[:-1] + ',"hotel":[]}')
# Product response without its closing brace, for pairs assembled by SSI (pair.shtml)
self.write('product-part/%s.part' % code, body[:-1])
self.exp.write('/_api/pair.shtml', '<!--#include virtual="/_api/product-part/${REDIRECT_API_PRODUCT}.part" -->'
',"hotel":<!--#include virtual="/_api/hotel/${REDIRECT_API_HOTEL}.json" -->}')
written = set()
for product_uid, hotel_uid in pairs:
for pc in product_codes.get(product_uid, ()):
for hc in hotel_codes.get(hotel_uid, ()):
if pc in products and hc in hotels and (pc, hc) not in written:
written.add((pc, hc))
self.write('product/%s/%s.json' % (pc, hc), products[pc][:-1] + ',"hotel":' + hotels[hc] + '}')
self.stats.update(hotels=len(hotels), products=len(products), pairs=len(written))
# The assembled files must be byte-identical to what TYPO3 answers
sample = sorted(written)[::max(1, len(written) // 40)] + [(pc, 'NO-SUCH-HOTEL') for pc in sorted(p for p in products if p != '@empty')[:5]]
for pc, hc in sample:
status, body = self.get('product?product=%s&hotel=%s' % (pc, urllib.parse.quote(hc)))
expected = products[pc][:-1] + (',"hotel":' + hotels[hc] if hc in hotels else ',"hotel":[]') + '}'
if status != 200 or body != expected:
self.exp.error('api: assembled /api/product?product=%s&hotel=%s differs from TYPO3' % (pc, hc))
# Error bodies exactly as ApiMiddleware sends them
for name, query in (('hotel-400', 'hotel'), ('hotel-404', 'hotel?hotel=NO-SUCH-CODE'),
('product-400', 'product'), ('product-404', 'product?product=NO-SUCH-CODE')):
self.write('error/%s.json' % name, self.get(query)[1])
self.write('error/403.json', self.exp.fetch(self.exp.dev + '/api/hotel?hotel=x')[1])
return API_RULES.replace('@KEY@', self.key)
API_HEADER = r'''# TYPO3 backchannel API: error bodies per endpoint, as ApiMiddleware sends them
AddCharset utf-8 .json
<If "%{REQUEST_URI} == '/api/hotel'">
ErrorDocument 400 /_api/error/hotel-400.json
ErrorDocument 403 /_api/error/403.json
ErrorDocument 404 /_api/error/hotel-404.json
</If>
<If "%{REQUEST_URI} == '/api/product'">
ErrorDocument 400 /_api/error/product-400.json
ErrorDocument 403 /_api/error/403.json
ErrorDocument 404 /_api/error/product-404.json
</If>
# Product + hotel pairs that were not pre-generated are assembled by SSI from the single responses
<IfModule mod_include.c>
<Files "pair.shtml">
Options +Includes
SetOutputFilter INCLUDES
ForceType "application/json; charset=utf-8"
</Files>
</IfModule>
'''
API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JSON snapshots taken from
# TYPO3 at export time; same status codes and bodies as ApiMiddleware
RewriteCond %{ENV:REDIRECT_STATUS} ^$
RewriteCond %{IS_SUBREQ} false
RewriteRule ^_api/ - [R=404,L]
RewriteCond %{HTTP:X-Api-Key} !=@KEY@
RewriteCond %{QUERY_STRING} !(^|&)key=@KEY@(&|$)
RewriteRule ^api/(hotel|product)$ - [R=403,L]
# Only file-safe codes are taken over; anything else leaves the variable unset (404 / "hotel": [])
RewriteCond %{QUERY_STRING} (^|&)hotel=([A-Za-z0-9_-]+)(&|$)
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:%2]
RewriteCond %{QUERY_STRING} (^|&)product=([A-Za-z0-9_-]+)(&|$)
RewriteRule ^api/product$ - [E=API_PRODUCT:%2]
# An empty code matches a record with an empty code in TYPO3: snapshot file "@empty" (never a real code)
RewriteCond %{QUERY_STRING} (^|&)hotel=(&|$)
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:@empty]
RewriteCond %{QUERY_STRING} (^|&)product=(&|$)
RewriteRule ^api/product$ - [E=API_PRODUCT:@empty]
RewriteCond %{QUERY_STRING} !(^|&)hotel=
RewriteRule ^api/hotel$ - [R=400,L]
RewriteCond %{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)$
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
RewriteRule ^api/hotel$ _api/hotel/%{ENV:API_HOTEL}.json [L]
RewriteRule ^api/hotel$ - [R=404,L]
RewriteCond %{QUERY_STRING} !(^|&)product=
RewriteRule ^api/product$ - [R=400,L]
RewriteCond %{QUERY_STRING} (^|&)hotel=
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json -f
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json [L]
# Pair not pre-generated but both known: assembled by SSI (exactly what TYPO3 returns)
<IfModule mod_include.c>
RewriteCond %{QUERY_STRING} (^|&)hotel=
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
RewriteCond %{DOCUMENT_ROOT}/_api/product-part/%{ENV:API_PRODUCT}.part -f
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
RewriteRule ^api/product$ _api/pair.shtml [L]
</IfModule>
# Hotel unknown (or no mod_include): TYPO3 answers with "hotel": []
RewriteCond %{QUERY_STRING} (^|&)hotel=
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json -f
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json [L]
RewriteCond %{QUERY_STRING} !(^|&)hotel=
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}.json -f
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}.json [L]
RewriteRule ^api/product$ - [R=404,L]
'''
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
STATIC_REWRITE = r'''
# Static export: reject multipart requests (CVE-2026-109618 / -109619)
RewriteCond %{HTTP:Content-Type} multipart [NC]
RewriteRule ^ - [F,L]
# No PHP is ever served, also not via a symlinked fileadmin
# (/buchung/ is exempt: it redirects to the booking system further down)
RewriteCond %{REQUEST_URI} !^/buchung/
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
# Search does not work statically: any search page goes to the homepage
RewriteRule (^|/)suche/?$ / [R=302,L]
'''
HYBRID_HEADER = '''### Begin: Static export (hybrid) ###
# Generated by .ddev/static-export/export.py --hybrid from public/.htaccess.
# Files of the static export win; anything else goes to TYPO3 13 through index.php.
DirectoryIndex index.html
ErrorDocument 404 /404.html
Options -Indexes -MultiViews
### End: Static export (hybrid) ###
'''
HYBRID_REWRITE = r'''
# Reject multipart requests (CVE-2026-109618 / -109619); the search sends URL-encoded forms
RewriteCond %{HTTP:Content-Type} multipart [NC]
RewriteRule ^ - [F,L]
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
# (/buchung/ is exempt: it redirects to the booking system further down)
RewriteCond %{REQUEST_URI} !^/buchung/
RewriteCond %{ENV:REDIRECT_STATUS} ^$ [OR]
RewriteCond %{REQUEST_URI} !^/index\.php$
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
# TYPO3 13 backend and install tool stay closed to the public
RewriteRule ^typo3(/|$) - [F,L]
'''
HYBRID_CATCH_ALL = ''' # Not part of the static export (a directory without index.html counts as missing) => TYPO3 13
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME}/index.html !-f
RewriteRule ^.*$ %{ENV:CWD}index.php [QSA,L]
'''
def build_htaccess(hybrid=False):
"""TYPO3's .htaccess without the parts that hand requests to PHP (hybrid: with the hand-over)."""
with open(os.path.join(PUBLIC, '.htaccess'), encoding='utf-8') as fh:
text = fh.read()
text, sfc = re.subn(r'[ \t]*### Begin: StaticFileCache \(preparation\).*?### End: StaticFileCache ###\n', '', text, flags=re.S)
text, catch_all = re.subn(r'(?:[ \t]*#[^\n]*\n)*(?:[ \t]*RewriteCond %\{REQUEST_FILENAME\} !-[fdl]\n){3}[ \t]*RewriteRule \^\.\*\$ %\{ENV:CWD\}index\.php \[QSA,L\]\n',
lambda m: HYBRID_CATCH_ALL if hybrid else '', text)
rewrite = HYBRID_REWRITE if hybrid else STATIC_REWRITE
text, engine = re.subn(r'(\n[ \t]*RewriteEngine On\n)', r'\1' + (rewrite + REDIRECTS_PLACEHOLDER).replace('\\', '\\\\'), text, count=1)
leftover = re.sub(r'#[^\n]*', '', text.replace(HYBRID_CATCH_ALL, '') if hybrid else text)
if (sfc, catch_all, engine) != (1, 1, 1) or re.search(r'RewriteRule[^\n]*index\.php', leftover):
sys.exit('public/.htaccess changed shape, adjust build_htaccess() (sfc=%d, catch-all=%d, engine=%d)' % (sfc, catch_all, engine))
return (HYBRID_HEADER if hybrid else STATIC_HEADER) + text
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('--site', action='append', help='site identifier from config/sites (repeatable, default: all)')
ap.add_argument('--url', action='append', help='export only these URLs (dev URLs) instead of the sitemap')
ap.add_argument('--no-follow', action='store_true', help='do not follow internal links beyond the start URLs')
ap.add_argument('--out', default=os.path.join(APP_ROOT, 'var', 'static-export'))
ap.add_argument('--workers', type=int, default=8)
ap.add_argument('--hybrid', action='store_true', help='leave product pages and search to TYPO3 13 (see module doc)')
ap.add_argument('--typo3-current', help='hybrid: absolute path of the TYPO3 13 "current" release symlink on the server')
ap.add_argument('--live-ajax', action='store_true', help='hybrid: keep AJAX URLs that TYPO3 13 serves instead of pre-fetching them')
args = ap.parse_args()
if args.hybrid and not args.typo3_current:
ap.error('--hybrid needs --typo3-current')
sys.stdout.reconfigure(line_buffering=True)
all_sites = load_sites()
selected = {k: v for k, v in all_sites.items() if not args.site or k in args.site}
if not selected:
sys.exit('No matching site in %s' % SITES_DIR)
htaccess = build_htaccess(args.hybrid)
failed = False
for name, site in selected.items():
out = os.path.join(args.out, name)
shutil.rmtree(out, ignore_errors=True)
exp = Exporter(name, site, all_sites, out, follow=not args.no_follow and not args.url, workers=args.workers,
hybrid=args.hybrid, live_ajax=args.live_ajax)
print('== %s: %s → %s' % (name, site['dev'], site['prod']))
if args.url:
start = [u for u in args.url if u.startswith(site['dev'])]
else:
start = [site['dev'] + '/'] + exp.sitemap()
if not args.hybrid:
# Linked from nowhere (handed out to groups and customers); in hybrid mode TYPO3 13 serves them
routes = route_urls(site)
start += routes
if routes:
print(' route URLs (/gruppe/, /reiseinformationen/): %d' % len(routes))
exp.special_files()
exp.crawl(start)
rules = exp.redirects()
site_htaccess = htaccess
if args.hybrid:
# TYPO3 13 behind index.php
# /_assets/ is served by the stub too (no symlink: not every vhost follows one)
exp.write('/index.php', FRONT_CONTROLLER_STUB % args.typo3_current.rstrip('/'))
if name in API_SITES and not args.url and not args.hybrid:
api = ApiExporter(exp)
rules = api.run() + rules
site_htaccess = API_HEADER + htaccess
print(' api: %d hotels, %d products, %d product/hotel pairs, codes skipped (not file-safe): %s'
% (api.stats['hotels'], api.stats['products'], api.stats['pairs'], ', '.join(api.stats['skipped_codes']) or '-'))
if exp.crawl_redirects:
rules += '\t# Redirects returned by TYPO3 while crawling\n' + '\n'.join(sorted(exp.crawl_redirects)) + '\n'
exp.write('/.htaccess', site_htaccess.replace(REDIRECTS_PLACEHOLDER, '\n' + rules))
copied, missing = exp.copy_assets()
print(' pages: %d, ajax tables: %d, redirects: %d (%d skipped, see %s-redirects-skipped.txt), assets: %d copied, %d missing'
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
if args.hybrid:
print(' left to TYPO3 13: %d pages' % exp.stats['dynamic'])
for path in sorted(missing)[:20]:
print(' missing asset: %s' % path)
for msg in exp.errors[:20]:
print(' error: %s' % msg)
if len(exp.errors) > 20:
print(' ... %d more errors' % (len(exp.errors) - 20))
failed = failed or bool(exp.errors)
sys.exit(1 if failed else 0)
if __name__ == '__main__':
main()