#!/usr/bin/env python3 """Static export of the TYPO3 sites running in DDEV. Crawls each site (sitemap + internal links), pre-fetches the AJAX dates and price tables of product pages, rewrites *.ddev.site hosts to the production domains and copies every referenced static asset. The result is a set of plain document roots that run without PHP. With --hybrid the document roots sit in front of TYPO3 13 instead: product pages and the search are left out (TYPO3 13 serves them through index.php), the search UI stays (the search bar is refreshed from TYPO3 13), and the API is answered live. Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR] [--hybrid --typo3-current PATH [--live-ajax]] """ import argparse import collections import concurrent.futures import hashlib import html import http.client import os import re import shutil import subprocess import sys import threading import urllib.error import urllib.parse import urllib.request APP_ROOT = os.environ.get('DDEV_APPROOT', '/var/www/html') PUBLIC = os.path.join(APP_ROOT, 'public') SITES_DIR = os.path.join(APP_ROOT, 'config', 'sites') # Hosts that pull from the production origin; their paths must exist in the export CDN_HOSTS = {'cdn.ep-reisen.de'} # Any attribute pointing at TYPO3's AJAX page type (product tables, HTMX teaser # popups, ...) is pre-fetched; JS may append "&pa=..." to these URLs AJAX_PATH = '/ajax/' # Forms posting to these TYPO3 plugins (contact, enquiry, application, booking, newsletter) # cannot work statically and are replaced by a contact notice CONTACT_FORM_RE = re.compile(r']*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I) # Hybrid: EXT:form plugins (ep_theme Templates/Form/Render.html:
) are # loaded from TYPO3 13 instead, from /form (page type 1704, renders the page's form only). # htmx swaps the placeholder on load and the form posts back there. ?ref= (the page a visitor came # from) is passed along for the enquiry e-mails. Until the form arrives the placeholder shows the # spinner of the form islands (ep_theme Partials/Form/Spinner.html, here in the text colour: the # theme colours come with the island). The contact notice is shown without JavaScript and, taken # from the noscript element, when TYPO3 13 does not answer. FORM_PLUGIN_RE = re.compile(r'
(?=\s*]*\baction="[^"]*tx_form_formframework)', re.I) FORM_ISLAND_FALLBACK = "this.removeAttribute('style'); this.innerHTML = this.querySelector('noscript').textContent" FORM_SPINNER = ( '' '' + ''.join('' '' '' '' % (begin, begin) for begin in ('0s', '-0.9s')) + '') FORM_ISLAND = ('
' + FORM_SPINNER + '
') # Hybrid: snow report lists (ep_theme Templates/Snowreport/List.html: a column of rows starting # with flag and region, in the spacing wrapper of ep:container if the element has spacing) are # loaded from TYPO3 13, from /snowreport (page type 1705, renders the page's snow reports # only, each in id="snowreport-" around the spacing wrapper). The export keeps an # empty placeholder only: no outdated snow depths, and nothing without JavaScript. When TYPO3 13 # does not answer or no longer has that list, the placeholder is removed. SNOWREPORT_BOX_RE = re.compile( r'(?<=)\s*
' r'(?=(?:\s*
)?\s*
' r'\s*
)', re.I) CONTENT_ANCHOR_RE = re.compile(r'$') SNOWREPORT_ISLAND = ('
') # Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them, # and links to the search page (menu icon, "Alle anzeigen" with preset filters) SEARCH_ELEMENTS = ( ('div', re.compile(r']*\bdata-controller="searchbar"[^>]*>')), ('div', re.compile(r']*\bdata-controller="mobile-search"[^>]*>')), ('button', re.compile(r']*\bdata-overlay-toggle-id-value="(?:mobile-)?search"[^>]*>')), ('a', re.compile(r']*\bhref="[^"]*/suche/(?:\?[^"]*)?"[^>]*>')), ) SEARCH_PAGE_RE = re.compile(r'/suche/$') # Hybrid: the search UI stays, the search itself is served by TYPO3 13. The search bar # (ep_theme Templates/Searchbar/Index.html) is replaced on load by the current one from TYPO3 13, # from searchbar or searchbar-collapsed (page types 1706/1707, the same on every # page). Until then, and when TYPO3 13 does not answer, the exported bar stays: its form is # rewritten to the GET form of TYPO3 13 (fields as SearchController::SEARCHBAR_PARAMETERS), # since TYPO3 13 has no processSearch and would reject the signed fields of TYPO3 10. HYBRID_SEARCH_ELEMENTS = () # The whole start tag: its data-action contains "->" SEARCHBAR_RE = re.compile(r'"]|"[^"]*")*?\sdata-controller="searchbar"(?:[^>"]|"[^"]*")*>') SEARCHBAR_FORM_RE = re.compile(r'') SEARCHBAR_SIGNED_FIELDS_RE = re.compile(r'
(?:\s*

{lead}

{text}

{phone} {email}

''' def replace_elements(text, tag, start_re, replacement): """Replace each element whose start tag matches, including its balanced content. replacement may be a function of the start tag match and the element, returning the new markup.""" token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I) out, pos, count = [], 0, 0 for m in start_re.finditer(text): if m.start() < pos: continue # nested inside an element already replaced depth, end = 1, None for t in token_re.finditer(text, m.end()): depth += -1 if t.group(1) else 1 if depth == 0: end = t.end() break if end is None: continue # unbalanced markup, leave it alone out += [text[pos:m.start()], replacement(m, text[m.start():end]) if callable(replacement) else replacement] pos, count = end, count + 1 out.append(text[pos:]) return ''.join(out), count URL_ATTR_RE = re.compile(r'(?:href|src|srcset|data-src|data-srcset|content|poster|action)="([^"]*)"') CSS_URL_RE = re.compile(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)') ASSET_EXT_RE = re.compile(r'\.(css|js|mjs|json|map|svg|png|jpe?g|gif|webp|avif|ico|woff2?|ttf|eot|otf|pdf|mp4|webm|txt|xml|webmanifest)$', re.I) class NoRedirect(urllib.request.HTTPRedirectHandler): """Report redirects instead of following them, so they become rewrite rules.""" def redirect_request(self, req, fp, code, msg, headers, newurl): return None OPENER = urllib.request.build_opener(NoRedirect) def rewrite_rule(path, location, status): pattern = re.escape(urllib.parse.unquote(path).strip('/')).replace('\\ ', '\\x20') target = re.sub(r'([$%])', r'\\\1', location) return '\tRewriteRule "^%s/?$" "%s" [R=%d,L,NE]' % (pattern, target, status) def load_sites(only=None): """Read base and Development base variant from each site config.""" sites = {} for name in sorted(os.listdir(SITES_DIR)): path = os.path.join(SITES_DIR, name, 'config.yaml') if not os.path.isfile(path) or (only and name not in only): continue with open(path, encoding='utf-8') as fh: text = fh.read() prod = re.search(r"^base:\s*'?([^'\s]+)", text, re.M) dev = re.search(r"base:\s*'?(https://[^'\s]+\.ddev\.site)'?\s*\n\s*condition:\s*'applicationContext == \"Development\"'", text) if prod and dev and prod.group(1).startswith('http'): root = re.search(r'^rootPageId:\s*(\d+)', text, re.M) sites[name] = {'prod': prod.group(1).rstrip('/'), 'dev': dev.group(1).rstrip('/'), 'root': int(root.group(1)) if root else 0, 'languages': site_languages(text)} return sites def site_languages(text): """languageId => base path prefix ('' for '/', '/en' for '/en/') of a site config.""" section = re.search(r'^languages:\n((?:[ \t-].*\n|\n)*)', text, re.M) languages = {} for item in re.split(r'^ -[ \t]*\n?', section.group(1) if section else '', flags=re.M): language_id = re.search(r"^\s*languageId:\s*'?(\d+)", item, re.M) base = re.search(r"^\s{4}base:\s*'?([^'\s]+)", item, re.M) if language_id and base and base.group(1).startswith('/'): languages[int(language_id.group(1))] = base.group(1).rstrip('/') return languages class Exporter: def __init__(self, name, site, all_sites, out_dir, follow, workers, hybrid=False, live_ajax=False): self.site = site self.hybrid = hybrid self.live_ajax = live_ajax self.dynamic = dynamic_paths(site) if hybrid else set() self.route_prefixes = set(route_pages(site).values()) if hybrid else set() self.contact = CONTACTS.get(name, DEFAULT_CONTACT) self.dev = site['dev'] self.dev_host = urllib.parse.urlsplit(self.dev).netloc # Editors often paste absolute production URLs; those are internal links too self.prod_host = urllib.parse.urlsplit(site['prod']).netloc self.ajax_results = {} self.ajax_locks = {} self.out = out_dir self.follow = follow self.workers = workers # dev → prod for every site, so cross-site links come out right too self.host_map = [(s['dev'], s['prod']) for s in all_sites.values()] # Some pages get their AJAX URLs below a parent slug (/reisen-fuer-gruppen/ajax/html?...) self.ajax_attr_re = re.compile(r'([a-zA-Z][\w:.-]*)="((?:%s)?(?:/[\w-]+)*%s[^"]*)"' % (re.escape(self.dev), re.escape(AJAX_PATH))) self.seen = set() self.assets = set() self.lock = threading.Lock() self.errors = [] self.crawl_redirects = [] self.follow_redirects = follow self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'form_islands': 0, 'snowreports': 0, 'searchbars': 0, 'search': 0, 'dynamic': 0} # --- helpers ----------------------------------------------------------- def fetch(self, url, extra_headers=None, log_errors=True): """Return status, body and (for redirects) the Location header.""" headers = {'User-Agent': 'ep-static-export'} headers.update(extra_headers or {}) parts = urllib.parse.urlsplit(url) if parts.scheme == 'https' and parts.hostname.endswith('.ddev.site'): # Talk to the container's own web server, bypassing ddev-router (works for hostnames # added without "ddev restart"); TYPO3 still sees HTTPS via the forwarded header url = urllib.parse.urlunsplit(('http', '127.0.0.1') + tuple(parts[2:])) headers.update({'Host': parts.hostname, 'X-Forwarded-Proto': 'https'}) req = urllib.request.Request(url, headers=headers) try: with OPENER.open(req, timeout=60) as res: return res.status, res.read().decode('utf-8', 'replace'), None except urllib.error.HTTPError as e: return e.code, e.read().decode('utf-8', 'replace'), e.headers.get('Location') except Exception as e: # noqa: BLE001 - report and continue if log_errors: self.error('%s: %s' % (url, e)) return 0, '', None def error(self, msg): with self.lock: self.errors.append(msg) def write(self, rel_path, content): path = os.path.join(self.out, rel_path.lstrip('/')) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'w', encoding='utf-8') as fh: fh.write(content) def to_prod(self, text): for dev, prod in self.host_map: text = text.replace(dev, prod) text = text.replace(dev.replace('/', '\\/'), prod.replace('/', '\\/')) text = text.replace(urllib.parse.quote(dev, safe=''), urllib.parse.quote(prod, safe='')) return text def local_path(self, url): """Path on this site for a URL, or None if it points elsewhere.""" parts = urllib.parse.urlsplit(html.unescape(url)) if parts.scheme and parts.scheme not in ('http', 'https'): return None if parts.netloc and parts.netloc not in (self.dev_host, self.prod_host) and parts.netloc not in CDN_HOSTS: return None if not parts.path.startswith('/') or parts.path.startswith('//'): return None return parts def collect(self, text): """Collect asset paths and (if following) internal page links.""" links = set() for raw in URL_ATTR_RE.findall(text) + CSS_URL_RE.findall(text): for candidate in raw.split(','): # srcset candidate = candidate.strip().split(' ')[0] parts = self.local_path(candidate) if not parts: continue path = urllib.parse.unquote(parts.path) if ASSET_EXT_RE.search(path): self.assets.add(path) elif self.follow and not parts.query and parts.netloc in ('', self.dev_host, self.prod_host) \ and not path.startswith(('/typo3', AJAX_PATH)): links.add(self.dev + path) return links # --- pages and AJAX tables --------------------------------------------- def ajax_file(self, url): """Fetch an AJAX table, store it, return its static URL.""" url = html.unescape(url) if url.startswith('/'): url = self.dev + url name = '/_ajax/%s.html' % hashlib.sha1(url.replace(self.dev, '').encode()).hexdigest() with self.lock: name_lock = self.ajax_locks.setdefault(name, threading.RLock()) # Every reference waits for the first fetch, so a failed table keeps its original URL everywhere with name_lock: if name not in self.ajax_results: # Set before fetching: a response referencing itself must not recurse. # The JS appends "&pa=..."; the dummy query keeps that harmless self.ajax_results[name] = name + '?s=1' status, body, _ = self.fetch(url) if status != 200: self.error('%s → %s' % (url, status)) self.ajax_results[name] = url else: self.write(name, self.to_prod(self.process(self.strip_dynamic(body, '')))) with self.lock: self.stats['ajax'] += 1 return self.ajax_results[name] def strip_dynamic(self, text, path): """Swap contact forms for a contact notice and drop the search UI.""" phone, tel, email = self.contact lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de'] notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email) islands = snowreports = searchbars = 0 if self.hybrid and path: text, searchbars = replace_elements(text, 'div', SEARCHBAR_RE, lambda m, element: self.searchbar_island(path, m, element)) island = FORM_ISLAND.format(url=html.escape(path.rstrip('/') + '/form'), notice=notice) text, islands = replace_elements(text, 'div', FORM_PLUGIN_RE, island) text, snowreports = replace_elements(text, 'div', SNOWREPORT_BOX_RE, lambda m, element: self.snowreport_island(text, path, m, element)) text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice) removed = 0 for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS): text, n = replace_elements(text, tag, start_re, '') removed += n with self.lock: self.stats['forms'] += forms self.stats['form_islands'] += islands self.stats['snowreports'] += snowreports self.stats['searchbars'] += searchbars self.stats['search'] += removed return text def searchbar_island(self, path, m, element): """The exported search bar with the GET form of TYPO3 13, replaced on load by the current bar from TYPO3 13 (open or collapsed, as exported).""" element, forms = SEARCHBAR_FORM_RE.subn( lambda f: '' % html.escape(html.unescape(f.group(1)).split('?')[0]), element) # host: see process() element, signed = SEARCHBAR_SIGNED_FIELDS_RE.subn('', element) element, fields = SEARCHBAR_FIELD_RE.subn(r'name="\1"', element) if (forms, signed, fields) != (1, 1, 6): self.error('%s: search bar form not as expected (form %d, signed fields %d, fields %d), left out' % (path, forms, signed, fields)) return '' variant = 'searchbar-collapsed' if 'data-searchbar-collapsed-value="true"' in element else 'searchbar' url = ('/en/' if path.startswith('/en/') else '/') + variant start = m.group(0) return start[:-1] + SEARCHBAR_ISLAND_ATTRIBUTES.format(url=url) + '>' + element[len(start):] def snowreport_island(self, text, path, m, element): """The placeholder loading the snow report box from TYPO3 13, for the content element of the anchor right before it; without one the box stays as exported.""" anchor = CONTENT_ANCHOR_RE.search(text, max(0, m.start() - 32), m.start()) if not anchor: self.error('%s: snow report without content element anchor' % path) return element return SNOWREPORT_ISLAND.format(url=html.escape(path.rstrip('/') + '/snowreport'), uid=anchor.group(1)) def process(self, text): """Replace AJAX table URLs (recursively fetching them) and collect assets.""" text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else '%s="%s"' % (m.group(1), html.escape(self.ajax_file(m.group(2)))), text) self.collect(text) return text def is_live_ajax(self, url): """Hybrid --live-ajax: keep the URL when TYPO3 13 serves the action.""" params = dict(LIVE_AJAX_PARAM_RE.findall(html.unescape(url))) return self.live_ajax and (params.get('controller'), params.get('action')) in LIVE_AJAX def is_dynamic(self, path): """Hybrid: product pages (with their /detail/ variants) and search pages belong to TYPO3 13.""" path = path.rstrip('/') + '/' return (path in self.dynamic or any(path.startswith(p + 'detail/') for p in self.dynamic) or any(path.startswith(p) and path != p for p in self.route_prefixes)) def safe_page(self, url): try: return self.page(url) except Exception as e: # noqa: BLE001 - one broken page must not stop the export self.error('%s: %s: %s' % (url, type(e).__name__, e)) return set() def page(self, url): status, body, location = self.fetch(url) if status in (301, 302, 303, 307, 308) and location: location = urllib.parse.urljoin(url, location) with self.lock: self.crawl_redirects.append(rewrite_rule(urllib.parse.urlsplit(url).path, self.to_prod(location), status)) internal = location.startswith(self.dev + '/') and '?' not in location return {location} if internal and self.follow_redirects else set() if status != 200: self.error('%s → %s' % (url, status)) return set() path = urllib.parse.urlsplit(url).path if self.hybrid and self.is_dynamic(path): # Served by TYPO3 13 through index.php; only its links are followed with self.lock: self.stats['dynamic'] += 1 return self.collect(body) if not self.hybrid and SEARCH_PAGE_RE.search(path): # Search does not work statically: send visitors to the homepage instead with self.lock: self.crawl_redirects.append(rewrite_rule(path, self.site['prod'] + '/', 302)) return set() # Extensionless paths become directories, so /a and /a/b/ can both exist target = path if re.search(r'\.[a-z0-9]+$', path, re.I) else path.rstrip('/') + '/index.html' body = self.strip_dynamic(body, path) links = self.collect(body) self.write(target, self.to_prod(self.process(body))) with self.lock: self.stats['pages'] += 1 return links def sitemap(self): """Export sitemap index and children as plain files, return page URLs.""" status, index, _ = self.fetch(self.dev + '/sitemap.xml') if status != 200: print(' note: no sitemap.xml (%s), crawling from / only' % status) return [] urls = [] for loc in re.findall(r'([^<]+)', index): child_url = html.unescape(loc) name = urllib.parse.parse_qs(urllib.parse.urlsplit(child_url).query).get('sitemap', ['pages'])[0] _, child, _ = self.fetch(child_url) urls += [html.unescape(u) for u in re.findall(r'([^<]+)', child)] self.write('/sitemap-%s.xml' % name, self.to_prod(child)) index = index.replace(loc, self.dev + '/sitemap-%s.xml' % name) self.write('/sitemap.xml', self.to_prod(index)) return [u for u in urls if u.startswith(self.dev)] def crawl(self, start): queue = list(dict.fromkeys(start)) self.seen.update(queue) with concurrent.futures.ThreadPoolExecutor(self.workers) as pool: while queue: batch, queue = queue, [] for links in pool.map(self.safe_page, batch): for link in links: if link not in self.seen: self.seen.add(link) queue.append(link) def redirects(self): """Backend redirects (sys_redirect) as RewriteRules, targets resolved by TYPO3 itself.""" host = urllib.parse.urlsplit(self.site['prod']).netloc where = "deleted=0 AND disabled=0 AND source_host IN ('%s', '*')" % host try: out = subprocess.run( ['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', 'SELECT source_path FROM sys_redirect WHERE %s AND is_regexp=0; ' 'SELECT COUNT(*) FROM sys_redirect WHERE %s AND is_regexp=1' % (where, where)], capture_output=True, text=True, check=True).stdout.splitlines() except (OSError, subprocess.CalledProcessError) as e: self.error('sys_redirect: %s' % e) return '' paths, regexp = sorted(set(out[:-1])), int(out[-1]) skipped = ['regexp redirects (not exported): %d' % regexp] if regexp else [] def resolve(path): if '?' in path: return path, 0, 'query string in source path' conn = http.client.HTTPConnection('127.0.0.1', 80, timeout=60) try: conn.request('GET', urllib.parse.quote(path, safe="/%:@!$&'()*+,;=-._~"), headers={'Host': host}) res = conn.getresponse() res.read() return path, res.status, res.getheader('Location') finally: conn.close() rules = ['\t# Redirects from the TYPO3 backend (sys_redirect), resolved by TYPO3 at export time'] with concurrent.futures.ThreadPoolExecutor(self.workers) as pool: for path, status, location in pool.map(resolve, paths): if status not in (301, 302, 303, 307, 308) or not location: # Mostly dead entries (target page gone) that 404 in production already skipped.append('%s → %s %s' % (path, status, location or '')) continue # Resolve like a browser: TYPO3 sends "https:/path" for relative targets location = urllib.parse.urljoin(self.dev + path, location) if location.rstrip('/') == (self.dev + path).rstrip('/'): # TYPO3 matches "/a" for "/a/" too, so a redirect to the page at "/a/" loops skipped.append('%s → %s (redirects to itself)' % (path, location)) continue target = urllib.parse.urlsplit(location) if target.netloc == self.dev_host and ASSET_EXT_RE.search(target.path): self.assets.add(urllib.parse.unquote(target.path)) rules.append(rewrite_rule(path, self.to_prod(location), status)) self.stats['redirects'] = len(rules) - 1 self.stats['redirects_skipped'] = len(skipped) with open(self.out.rstrip('/') + '-redirects-skipped.txt', 'w', encoding='utf-8') as fh: fh.write('\n'.join(skipped) + '\n') return '\n'.join(rules) + '\n' def special_files(self): status, body, _ = self.fetch(self.dev + '/robots.txt') if status == 200: self.write('/robots.txt', self.to_prod(body)) # TYPO3 error page, served by Apache via ErrorDocument 404 status, body, _ = self.fetch(self.dev + '/static-export-not-found/') if status == 404 and body: self.write('/404.html', self.to_prod(self.process(self.strip_dynamic(body, '/')))) # --- static assets ----------------------------------------------------- def copy_assets(self): pending, copied, missing = list(self.assets), set(), [] while pending: path = pending.pop() if path in copied: continue copied.add(path) src = os.path.join(PUBLIC, path.lstrip('/')) if not os.path.isfile(src): missing.append(path) continue dst = os.path.join(self.out, path.lstrip('/')) os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copy2(src, dst) if path.endswith('.css'): # fonts and images referenced from CSS with open(src, encoding='utf-8', errors='replace') as fh: for ref in CSS_URL_RE.findall(fh.read()): if ref.startswith(('data:', 'http', '#')): continue ref = urllib.parse.urlsplit(ref).path pending.append(os.path.normpath(os.path.join(os.path.dirname(path), ref))) return len(copied) - len(missing), missing STATIC_HEADER = '''### Begin: Static export ### # Generated by .ddev/static-export/export.py from public/.htaccess. # No PHP: TYPO3's index.php catch-all and StaticFileCache rules are removed. DirectoryIndex index.html ErrorDocument 404 /404.html Options -Indexes -MultiViews php_flag engine off ### End: Static export ### ''' # --- TYPO3 backchannel API (/api/hotel, /api/product) ---------------------------------------- # MyE&P calls the API on this site; ApiMiddleware itself answers on every host API_SITES = {'ep-reisen'} API_CODE_RE = re.compile(r'^[A-Za-z0-9_-]+$') def mysql_rows(sql): out = subprocess.run(['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', sql], capture_output=True, text=True, check=True).stdout return [line.split('\t') for line in out.splitlines() if line] # Records that only exist as URL arguments of one page (route enhancers), not linked anywhere: # (page uid, table, field, pattern the value must match or None) ROUTE_RECORDS = ( (1846, 'tx_epproducts_domain_model_product', 'path_segment', None), # /gruppe/{product} (2736, 'tx_epproducts_domain_model_travelinfo', 'path_segment', None), # /reiseinformationen/{name} (2736, 'tx_epproducts_domain_model_travelinfo', 'travel_code', r'[A-Z]+\d{6}'), # /reiseinformationen/{code} ) def page_tree_check(site): """Return a function telling whether a default-language page uid belongs to the site.""" parents = {int(uid): int(pid) for uid, pid in mysql_rows('SELECT uid, pid FROM pages WHERE sys_language_uid = 0')} def in_site(uid): for _ in range(100): if uid == site['root']: return True if uid not in parents: return False uid = parents[uid] return False return in_site def route_pages(site): """Slug (with trailing slash) of each route page of ROUTE_RECORDS on this site, by page uid.""" in_site = page_tree_check(site) base = urllib.parse.urlsplit(site['dev']).path.rstrip('/') pages = {} for uid in sorted({r[0] for r in ROUTE_RECORDS}): rows = mysql_rows('SELECT slug FROM pages WHERE uid = %d AND deleted = 0 AND hidden = 0' % uid) if rows and in_site(uid): pages[uid] = base + rows[0][0].rstrip('/') + '/' return pages def route_urls(site): """Start URLs for the records of ROUTE_RECORDS (not reachable through links or the sitemap).""" pages = route_pages(site) urls = set() for uid, table, field, pattern in ROUTE_RECORDS: if uid not in pages: continue for (value,) in mysql_rows("SELECT DISTINCT %s FROM %s WHERE deleted = 0 AND hidden = 0 AND %s <> ''" % (field, table, field)): if pattern is None or re.fullmatch(pattern, value): urls.add(site['dev'].rstrip('/') + pages[uid] + urllib.parse.quote(value) + '/') return sorted(urls) def dynamic_paths(site): """Paths (with trailing slash) of the pages TYPO3 13 serves in hybrid mode: pages of this site holding the product detail or search result plugin, in every language of the site.""" in_site = page_tree_check(site) rows = mysql_rows( "SELECT DISTINCT p.slug, p.sys_language_uid, IF(p.sys_language_uid = 0, p.uid, p.l10n_parent) " "FROM pages p JOIN tt_content c ON c.pid = p.uid " "WHERE p.deleted = 0 AND p.hidden = 0 AND c.deleted = 0 AND c.hidden = 0 AND c.CType = 'list' " "AND c.list_type IN ('epproducts_product_detail', 'epproducts_searchresult')") base = urllib.parse.urlsplit(site['dev']).path.rstrip('/') paths = set() for slug, language, default_uid in rows: if int(language) in site['languages'] and in_site(int(default_uid)): paths.add(base + site['languages'][int(language)] + slug.rstrip('/') + '/') return paths FRONT_CONTROLLER_STUB = """/...) come from the release as well: Apache does not // follow a symlink from the docroot into the release on every vhost, PHP does. $path = rawurldecode((string)strtok($_SERVER['REQUEST_URI'] ?? '', '?')); if (str_starts_with($path, '/_assets/')) { $types = [ 'css' => 'text/css; charset=utf-8', 'js' => 'text/javascript; charset=utf-8', 'json' => 'application/json', 'map' => 'application/json', 'svg' => 'image/svg+xml', 'png' => 'image/png', 'jpg' => 'image/jpeg', 'jpeg' => 'image/jpeg', 'gif' => 'image/gif', 'webp' => 'image/webp', 'avif' => 'image/avif', 'ico' => 'image/x-icon', 'woff' => 'font/woff', 'woff2' => 'font/woff2', 'ttf' => 'font/ttf', 'otf' => 'font/otf', 'eot' => 'application/vnd.ms-fontobject', ]; $file = realpath($release . '/public' . $path); $type = $types[strtolower(pathinfo((string)$file, PATHINFO_EXTENSION))] ?? null; // realpath() resolves the publishing symlinks; anything outside the release is refused if ($file === false || $type === null || !is_file($file) || !str_starts_with($file, realpath($release) . '/')) { http_response_code(404); exit; } header('Content-Type: ' . $type); header('Content-Length: ' . filesize($file)); header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($file)) . ' GMT'); // Content-hashed build files never change under their name (as in the .htaccess for static files). // The Expires header keeps mod_expires from adding a second, conflicting max-age. $maxAge = preg_match('/\\.[0-9a-f]{8}\\.\\w+$/', $file) ? 31536000 : 86400; header('Cache-Control: public, max-age=' . $maxAge . ($maxAge === 31536000 ? ', immutable' : '')); header('Expires: ' . gmdate('D, d M Y H:i:s', time() + $maxAge) . ' GMT'); readfile($file); exit; } require $release . '/public/index.php'; """ def cms_api_key(): php = '$c = include "%s/public/typo3conf/LocalConfiguration.php"; echo $c["EXTENSIONS"]["ep_products"]["cmsApiKey"] ?? "";' % APP_ROOT key = subprocess.run(['php', '-r', php], capture_output=True, text=True, check=True).stdout.strip() if not API_CODE_RE.match(key): sys.exit('cmsApiKey missing or with characters the .htaccess rules cannot match literally') return key class ApiExporter: """Snapshot /api/hotel and /api/product as JSON files, answered by TYPO3 itself. TYPO3 data is frozen while nobody edits, so the responses are too. A product+hotel response is the product response with "hotel": appended (ApiMiddleware json-encodes one array), so pair files are assembled from the single responses and a sample is checked against TYPO3. Pairs that were not generated get the response TYPO3 gives for an unknown hotel. """ def __init__(self, exp): self.exp = exp self.key = cms_api_key() self.stats = {'hotels': 0, 'products': 0, 'pairs': 0, 'skipped_codes': []} def get(self, query): url = self.exp.dev + '/api/' + query for _ in range(3): # the first call of a record can be slow while TYPO3 renders its images status, body, _ = self.exp.fetch(url, {'X-Api-Key': self.key}, log_errors=False) if status: return status, body self.exp.error('api: %s failed 3 times' % query) return status, body def responses(self, endpoint, codes): """code -> body for every code TYPO3 answers with 200.""" codes = sorted(set(codes)) fetch = lambda c: self.get('%s?%s=%s' % (endpoint, endpoint, urllib.parse.quote(c))) with concurrent.futures.ThreadPoolExecutor(self.exp.workers) as pool: # Warm-up pass: responses change while TYPO3 is still generating resized images list(pool.map(fetch, codes)) results = pool.map(fetch, codes) return {c: body for c, (status, body) in zip(codes, results) if status == 200} def write(self, path, body): self.exp.write('/_api/' + path, self.exp.to_prod(body)) def run(self): hotel_codes = collections.defaultdict(set) for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_hotel WHERE deleted=0 AND code<>''"): hotel_codes[uid].add(code) aliases = mysql_rows("SELECT entity, code FROM tx_epproducts_domain_model_matchcode WHERE deleted=0 AND code<>''") for uid, code in aliases: hotel_codes[uid].add(code) product_codes = collections.defaultdict(set) for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_product WHERE deleted=0 AND code<>''"): product_codes[uid].add(code) pairs = mysql_rows( "SELECT uid_local, uid_foreign FROM tx_epproducts_product_hotel_mm" " UNION SELECT product, hotel FROM tx_epproducts_domain_model_date WHERE deleted=0" " UNION SELECT product, hotel FROM tx_epproducts_domain_model_hotel_mapping WHERE deleted=0 AND hidden=0") def safe(codes): ok = {c for c in codes if API_CODE_RE.match(c)} self.stats['skipped_codes'] += sorted(set(codes) - ok) return ok # ApiMiddleware matches matchcode aliases for products too (join on matchcode.entity) hotels = self.responses('hotel', safe({c for cs in hotel_codes.values() for c in cs})) products = self.responses('product', safe({c for cs in product_codes.values() for c in cs} | {c for _, c in aliases})) # An empty code is not "missing" for ApiMiddleware: it matches a record with an empty code for endpoint, found in (('hotel', hotels), ('product', products)): status, body = self.get('%s?%s=' % (endpoint, endpoint)) if status == 200: found['@empty'] = body for code, body in hotels.items(): self.write('hotel/%s.json' % code, body) for code, body in products.items(): self.write('product/%s.json' % code, body) self.write('product/%s/_unknown-hotel.json' % code, body[:-1] + ',"hotel":[]}') # Product response without its closing brace, for pairs assembled by SSI (pair.shtml) self.write('product-part/%s.part' % code, body[:-1]) self.exp.write('/_api/pair.shtml', '' ',"hotel":}') written = set() for product_uid, hotel_uid in pairs: for pc in product_codes.get(product_uid, ()): for hc in hotel_codes.get(hotel_uid, ()): if pc in products and hc in hotels and (pc, hc) not in written: written.add((pc, hc)) self.write('product/%s/%s.json' % (pc, hc), products[pc][:-1] + ',"hotel":' + hotels[hc] + '}') self.stats.update(hotels=len(hotels), products=len(products), pairs=len(written)) # The assembled files must be byte-identical to what TYPO3 answers sample = sorted(written)[::max(1, len(written) // 40)] + [(pc, 'NO-SUCH-HOTEL') for pc in sorted(p for p in products if p != '@empty')[:5]] for pc, hc in sample: status, body = self.get('product?product=%s&hotel=%s' % (pc, urllib.parse.quote(hc))) expected = products[pc][:-1] + (',"hotel":' + hotels[hc] if hc in hotels else ',"hotel":[]') + '}' if status != 200 or body != expected: self.exp.error('api: assembled /api/product?product=%s&hotel=%s differs from TYPO3' % (pc, hc)) # Error bodies exactly as ApiMiddleware sends them for name, query in (('hotel-400', 'hotel'), ('hotel-404', 'hotel?hotel=NO-SUCH-CODE'), ('product-400', 'product'), ('product-404', 'product?product=NO-SUCH-CODE')): self.write('error/%s.json' % name, self.get(query)[1]) self.write('error/403.json', self.exp.fetch(self.exp.dev + '/api/hotel?hotel=x')[1]) return API_RULES.replace('@KEY@', self.key) API_HEADER = r'''# TYPO3 backchannel API: error bodies per endpoint, as ApiMiddleware sends them AddCharset utf-8 .json ErrorDocument 400 /_api/error/hotel-400.json ErrorDocument 403 /_api/error/403.json ErrorDocument 404 /_api/error/hotel-404.json ErrorDocument 400 /_api/error/product-400.json ErrorDocument 403 /_api/error/403.json ErrorDocument 404 /_api/error/product-404.json # Product + hotel pairs that were not pre-generated are assembled by SSI from the single responses Options +Includes SetOutputFilter INCLUDES ForceType "application/json; charset=utf-8" ''' API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JSON snapshots taken from # TYPO3 at export time; same status codes and bodies as ApiMiddleware RewriteCond %{ENV:REDIRECT_STATUS} ^$ RewriteCond %{IS_SUBREQ} false RewriteRule ^_api/ - [R=404,L] RewriteCond %{HTTP:X-Api-Key} !=@KEY@ RewriteCond %{QUERY_STRING} !(^|&)key=@KEY@(&|$) RewriteRule ^api/(hotel|product)$ - [R=403,L] # Only file-safe codes are taken over; anything else leaves the variable unset (404 / "hotel": []) RewriteCond %{QUERY_STRING} (^|&)hotel=([A-Za-z0-9_-]+)(&|$) RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:%2] RewriteCond %{QUERY_STRING} (^|&)product=([A-Za-z0-9_-]+)(&|$) RewriteRule ^api/product$ - [E=API_PRODUCT:%2] # An empty code matches a record with an empty code in TYPO3: snapshot file "@empty" (never a real code) RewriteCond %{QUERY_STRING} (^|&)hotel=(&|$) RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:@empty] RewriteCond %{QUERY_STRING} (^|&)product=(&|$) RewriteRule ^api/product$ - [E=API_PRODUCT:@empty] RewriteCond %{QUERY_STRING} !(^|&)hotel= RewriteRule ^api/hotel$ - [R=400,L] RewriteCond %{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)$ RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f RewriteRule ^api/hotel$ _api/hotel/%{ENV:API_HOTEL}.json [L] RewriteRule ^api/hotel$ - [R=404,L] RewriteCond %{QUERY_STRING} !(^|&)product= RewriteRule ^api/product$ - [R=400,L] RewriteCond %{QUERY_STRING} (^|&)hotel= RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$ RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json -f RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json [L] # Pair not pre-generated but both known: assembled by SSI (exactly what TYPO3 returns) RewriteCond %{QUERY_STRING} (^|&)hotel= RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$ RewriteCond %{DOCUMENT_ROOT}/_api/product-part/%{ENV:API_PRODUCT}.part -f RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f RewriteRule ^api/product$ _api/pair.shtml [L] # Hotel unknown (or no mod_include): TYPO3 answers with "hotel": [] RewriteCond %{QUERY_STRING} (^|&)hotel= RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$ RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json -f RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json [L] RewriteCond %{QUERY_STRING} !(^|&)hotel= RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$ RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}.json -f RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}.json [L] RewriteRule ^api/product$ - [R=404,L] ''' REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n' STATIC_REWRITE = r''' # No PHP is ever served, also not via a symlinked fileadmin # (/buchung/ is exempt: it redirects to the booking system further down) RewriteCond %{REQUEST_URI} !^/buchung/ RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC] # Search does not work statically: any search page goes to the homepage RewriteRule (^|/)suche/?$ / [R=302,L] ''' HYBRID_HEADER = '''### Begin: Static export (hybrid) ### # Generated by .ddev/static-export/export.py --hybrid from public/.htaccess. # Files of the static export win; anything else goes to TYPO3 13 through index.php. DirectoryIndex index.html ErrorDocument 404 /404.html Options -Indexes -MultiViews ### End: Static export (hybrid) ### ''' HYBRID_REWRITE = r''' # No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below # (/buchung/ is exempt: it redirects to the booking system further down) RewriteCond %{REQUEST_URI} !^/buchung/ RewriteCond %{ENV:REDIRECT_STATUS} ^$ [OR] RewriteCond %{REQUEST_URI} !^/index\.php$ RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC] # TYPO3 13 backend and install tool stay closed to the public RewriteRule ^typo3(/|$) - [F,L] ''' HYBRID_CATCH_ALL = ''' # Not part of the static export (a directory without index.html counts as missing) => TYPO3 13 RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME}/index.html !-f RewriteRule ^.*$ %{ENV:CWD}index.php [QSA,L] ''' def build_htaccess(hybrid=False): """TYPO3's .htaccess without the parts that hand requests to PHP (hybrid: with the hand-over).""" with open(os.path.join(PUBLIC, '.htaccess'), encoding='utf-8') as fh: text = fh.read() text, sfc = re.subn(r'[ \t]*### Begin: StaticFileCache \(preparation\).*?### End: StaticFileCache ###\n', '', text, flags=re.S) text, catch_all = re.subn(r'(?:[ \t]*#[^\n]*\n)*(?:[ \t]*RewriteCond %\{REQUEST_FILENAME\} !-[fdl]\n){3}[ \t]*RewriteRule \^\.\*\$ %\{ENV:CWD\}index\.php \[QSA,L\]\n', lambda m: HYBRID_CATCH_ALL if hybrid else '', text) rewrite = HYBRID_REWRITE if hybrid else STATIC_REWRITE text, engine = re.subn(r'(\n[ \t]*RewriteEngine On\n)', r'\1' + (rewrite + REDIRECTS_PLACEHOLDER).replace('\\', '\\\\'), text, count=1) leftover = re.sub(r'#[^\n]*', '', text.replace(HYBRID_CATCH_ALL, '') if hybrid else text) if (sfc, catch_all, engine) != (1, 1, 1) or re.search(r'RewriteRule[^\n]*index\.php', leftover): sys.exit('public/.htaccess changed shape, adjust build_htaccess() (sfc=%d, catch-all=%d, engine=%d)' % (sfc, catch_all, engine)) return (HYBRID_HEADER if hybrid else STATIC_HEADER) + text def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--site', action='append', help='site identifier from config/sites (repeatable, default: all)') ap.add_argument('--url', action='append', help='export only these URLs (dev URLs) instead of the sitemap') ap.add_argument('--no-follow', action='store_true', help='do not follow internal links beyond the start URLs') ap.add_argument('--out', default=os.path.join(APP_ROOT, 'var', 'static-export')) ap.add_argument('--workers', type=int, default=8) ap.add_argument('--hybrid', action='store_true', help='leave product pages and search to TYPO3 13 (see module doc)') ap.add_argument('--typo3-current', help='hybrid: absolute path of the TYPO3 13 "current" release symlink on the server') ap.add_argument('--live-ajax', action='store_true', help='hybrid: keep AJAX URLs that TYPO3 13 serves instead of pre-fetching them') args = ap.parse_args() if args.hybrid and not args.typo3_current: ap.error('--hybrid needs --typo3-current') sys.stdout.reconfigure(line_buffering=True) all_sites = load_sites() selected = {k: v for k, v in all_sites.items() if not args.site or k in args.site} if not selected: sys.exit('No matching site in %s' % SITES_DIR) htaccess = build_htaccess(args.hybrid) failed = False for name, site in selected.items(): out = os.path.join(args.out, name) shutil.rmtree(out, ignore_errors=True) exp = Exporter(name, site, all_sites, out, follow=not args.no_follow and not args.url, workers=args.workers, hybrid=args.hybrid, live_ajax=args.live_ajax) print('== %s: %s → %s' % (name, site['dev'], site['prod'])) if args.url: start = [u for u in args.url if u.startswith(site['dev'])] else: start = [site['dev'] + '/'] + exp.sitemap() if not args.hybrid: # Linked from nowhere (handed out to groups and customers); in hybrid mode TYPO3 13 serves them routes = route_urls(site) start += routes if routes: print(' route URLs (/gruppe/, /reiseinformationen/): %d' % len(routes)) exp.special_files() exp.crawl(start) rules = exp.redirects() site_htaccess = htaccess if args.hybrid: # TYPO3 13 behind index.php # /_assets/ is served by the stub too (no symlink: not every vhost follows one) exp.write('/index.php', FRONT_CONTROLLER_STUB % args.typo3_current.rstrip('/')) if name in API_SITES and not args.url and not args.hybrid: api = ApiExporter(exp) rules = api.run() + rules site_htaccess = API_HEADER + htaccess print(' api: %d hotels, %d products, %d product/hotel pairs, codes skipped (not file-safe): %s' % (api.stats['hotels'], api.stats['products'], api.stats['pairs'], ', '.join(api.stats['skipped_codes']) or '-')) if exp.crawl_redirects: rules += '\t# Redirects returned by TYPO3 while crawling\n' + '\n'.join(sorted(exp.crawl_redirects)) + '\n' exp.write('/.htaccess', site_htaccess.replace(REDIRECTS_PLACEHOLDER, '\n' + rules)) copied, missing = exp.copy_assets() print(' pages: %d, ajax tables: %d, redirects: %d (%d skipped, see %s-redirects-skipped.txt), assets: %d copied, %d missing' % (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing))) print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search'])) if args.hybrid: print(' left to TYPO3 13: %d pages, forms loaded from TYPO3 13: %d, snow reports loaded from TYPO3 13: %d,' ' search bars refreshed from TYPO3 13: %d' % (exp.stats['dynamic'], exp.stats['form_islands'], exp.stats['snowreports'], exp.stats['searchbars'])) for path in sorted(missing)[:20]: print(' missing asset: %s' % path) for msg in exp.errors[:20]: print(' error: %s' % msg) if len(exp.errors) > 20: print(' ... %d more errors' % (len(exp.errors) - 20)) failed = failed or bool(exp.errors) sys.exit(1 if failed else 0) if __name__ == '__main__': main()