1018 lines
53 KiB
Python
1018 lines
53 KiB
Python
#!/usr/bin/env python3
|
|
"""Static export of the TYPO3 sites running in DDEV.
|
|
|
|
Crawls each site (sitemap + internal links), pre-fetches the AJAX dates and
|
|
price tables of product pages, rewrites *.ddev.site hosts to the production
|
|
domains and copies every referenced static asset. The result is a set of
|
|
plain document roots that run without PHP.
|
|
|
|
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
|
|
and the search are left out (TYPO3 13 serves them through index.php), the search
|
|
UI stays (the search bar is refreshed from TYPO3 13), and the API is answered live.
|
|
|
|
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
|
|
[--hybrid --typo3-current PATH [--live-ajax]]
|
|
"""
|
|
|
|
import argparse
|
|
import collections
|
|
import concurrent.futures
|
|
import hashlib
|
|
import html
|
|
import http.client
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
APP_ROOT = os.environ.get('DDEV_APPROOT', '/var/www/html')
|
|
PUBLIC = os.path.join(APP_ROOT, 'public')
|
|
SITES_DIR = os.path.join(APP_ROOT, 'config', 'sites')
|
|
|
|
# Hosts that pull from the production origin; their paths must exist in the export
|
|
CDN_HOSTS = {'cdn.ep-reisen.de'}
|
|
|
|
# Any attribute pointing at TYPO3's AJAX page type (product tables, HTMX teaser
|
|
# popups, ...) is pre-fetched; JS may append "&pa=..." to these URLs
|
|
AJAX_PATH = '/ajax/'
|
|
|
|
# Forms posting to these TYPO3 plugins (contact, enquiry, application, booking, newsletter)
|
|
# cannot work statically and are replaced by a contact notice
|
|
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
|
|
|
|
# Hybrid: EXT:form plugins (ep_theme Templates/Form/Render.html: <div class="py-8"><form ...>) are
|
|
# loaded from TYPO3 13 instead, from <page>/form (page type 1704, renders the page's form only).
|
|
# htmx swaps the placeholder on load and the form posts back there. ?ref= (the page a visitor came
|
|
# from) is passed along for the enquiry e-mails. Until the form arrives the placeholder shows the
|
|
# spinner of the form islands (ep_theme Partials/Form/Spinner.html, here in the text colour: the
|
|
# theme colours come with the island). The contact notice is shown without JavaScript and, taken
|
|
# from the noscript element, when TYPO3 13 does not answer.
|
|
FORM_PLUGIN_RE = re.compile(r'<div class="py-8">(?=\s*<form\b[^>]*\baction="[^"]*tx_form_formframework)', re.I)
|
|
FORM_ISLAND_FALLBACK = "this.removeAttribute('style'); this.innerHTML = this.querySelector('noscript').textContent"
|
|
FORM_SPINNER = (
|
|
'<svg style="width:4rem;height:4rem;opacity:.5" viewBox="0 0 44 44" xmlns="http://www.w3.org/2000/svg" stroke="currentColor">'
|
|
'<g fill="none" fill-rule="evenodd" stroke-width="2">'
|
|
+ ''.join('<circle cx="22" cy="22" r="1">'
|
|
'<animate attributeName="r" begin="%s" dur="1.8s" values="1; 20" calcMode="spline" keyTimes="0; 1" keySplines="0.165, 0.84, 0.44, 1" repeatCount="indefinite" />'
|
|
'<animate attributeName="stroke-opacity" begin="%s" dur="1.8s" values="1; 0" calcMode="spline" keyTimes="0; 1" keySplines="0.3, 0.61, 0.355, 1" repeatCount="indefinite" />'
|
|
'</circle>' % (begin, begin) for begin in ('0s', '-0.9s'))
|
|
+ '</g></svg>')
|
|
FORM_ISLAND = ('<div class="form-island" hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
|
|
' hx-vals=\'js:{{ref: new URLSearchParams(location.search).get("ref") || ""}}\''
|
|
' hx-on:htmx:response-error="' + FORM_ISLAND_FALLBACK + '"'
|
|
' hx-on:htmx:send-error="' + FORM_ISLAND_FALLBACK + '"'
|
|
' style="min-height:12rem;display:flex;align-items:center;justify-content:center">'
|
|
+ FORM_SPINNER + '<noscript>{notice}</noscript></div>')
|
|
|
|
# Hybrid: snow report lists (ep_theme Templates/Snowreport/List.html: a column of rows starting
|
|
# with flag and region, in the spacing wrapper of ep:container if the element has spacing) are
|
|
# loaded from TYPO3 13, from <page>/snowreport (page type 1705, renders the page's snow reports
|
|
# only, each in id="snowreport-<content uid>" around the spacing wrapper). The export keeps an
|
|
# empty placeholder only: no outdated snow depths, and nothing without JavaScript. When TYPO3 13
|
|
# does not answer or no longer has that list, the placeholder is removed.
|
|
SNOWREPORT_BOX_RE = re.compile(
|
|
r'(?<=</a>)\s*<div class="[^"]*">'
|
|
r'(?=(?:\s*<div class="flex flex-col space-y-2">)?\s*<div class="grid md:grid-cols-3 md:gap-y-0">'
|
|
r'\s*<div class="md:col-span-1 bg-ep-primary-dark p-2 flex items-start space-x-2">)', re.I)
|
|
CONTENT_ANCHOR_RE = re.compile(r'<a id="c(\d+)"></a>$')
|
|
SNOWREPORT_ISLAND = ('<div hx-get="{url}" hx-trigger="load" hx-select="#snowreport-{uid}" hx-swap="outerHTML"'
|
|
' hx-on:htmx:before-swap="event.detail.shouldSwap = event.detail.xhr.responseText'
|
|
'.includes(\'id="snowreport-{uid}"\'); event.detail.shouldSwap || this.remove()"'
|
|
' hx-on:htmx:send-error="this.remove()"></div>')
|
|
|
|
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
|
|
# and links to the search page (menu icon, "Alle anzeigen" with preset filters)
|
|
SEARCH_ELEMENTS = (
|
|
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
|
|
('div', re.compile(r'<div\b[^>]*\bdata-controller="mobile-search"[^>]*>')),
|
|
('button', re.compile(r'<button\b[^>]*\bdata-overlay-toggle-id-value="(?:mobile-)?search"[^>]*>')),
|
|
('a', re.compile(r'<a\b[^>]*\bhref="[^"]*/suche/(?:\?[^"]*)?"[^>]*>')),
|
|
)
|
|
SEARCH_PAGE_RE = re.compile(r'/suche/$')
|
|
|
|
# Hybrid: the search UI stays, the search itself is served by TYPO3 13. The search bar
|
|
# (ep_theme Templates/Searchbar/Index.html) is replaced on load by the current one from TYPO3 13,
|
|
# from <language root>searchbar or searchbar-collapsed (page types 1706/1707, the same on every
|
|
# page). Until then, and when TYPO3 13 does not answer, the exported bar stays: its form is
|
|
# rewritten to the GET form of TYPO3 13 (fields as SearchController::SEARCHBAR_PARAMETERS),
|
|
# since TYPO3 13 has no processSearch and would reject the signed fields of TYPO3 10.
|
|
HYBRID_SEARCH_ELEMENTS = ()
|
|
# The whole start tag: its data-action contains "->"
|
|
SEARCHBAR_RE = re.compile(r'<div\b(?:[^>"]|"[^"]*")*?\sdata-controller="searchbar"(?:[^>"]|"[^"]*")*>')
|
|
SEARCHBAR_FORM_RE = re.compile(r'<form data-searchbar-target="form" method="post" name="searchParams" action="([^"]*)">')
|
|
SEARCHBAR_SIGNED_FIELDS_RE = re.compile(r'<div>(?:\s*<input type="hidden" name="tx_epproducts_searchresult\[__[^>]*>)+\s*</div>')
|
|
SEARCHBAR_FIELD_RE = re.compile(r'name="tx_epproducts_searchresult\[searchParams\]\[(\w+)\]"')
|
|
SEARCHBAR_ISLAND_ATTRIBUTES = ' hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
|
|
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
|
|
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
|
|
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
|
|
# (404 on TYPO3 13 for popups pointing at another site)
|
|
LIVE_AJAX = {
|
|
('AjaxTable', 'datestable'), ('AjaxTable', 'pricetable'), ('AjaxTable', 'pricetableHtml'),
|
|
('AjaxTable', 'eventPricetable'),
|
|
('AjaxCalendar', 'priceTable'), ('AjaxCalendar', 'priceCalendar'),
|
|
('AjaxSearch', 'searchresult'), ('AjaxFilterPanel', 'index'), ('AjaxWatchlist', 'list'),
|
|
}
|
|
LIVE_AJAX_PARAM_RE = re.compile(r'tx_epproducts_ajax(?:%5B|\[)(controller|action)(?:%5D|\])=(\w+)')
|
|
|
|
# Contact shown instead of forms: display phone, tel: number, e-mail (as in each site's footer)
|
|
CONTACTS = {
|
|
'ep-reisen': ('0221 272276-0', '+492212722760', '[email protected]'),
|
|
'ski-boarderweek': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
|
'snowzone': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
|
'unichamp': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
|
'ser': ('0221 272276-41', '+4922127227641', '[email protected]'),
|
|
'alvac': ('0221 272276-30', '+4922127227630', '[email protected]'),
|
|
'cllt': ('0221 272276-30', '+4922127227630', '[email protected]'),
|
|
}
|
|
DEFAULT_CONTACT = CONTACTS['ep-reisen']
|
|
|
|
CONTACT_TEXT = {
|
|
'de': ('Dieses Formular ist vorübergehend nicht verfügbar.',
|
|
'Du hast Fragen oder möchtest buchen? Unser Kundenservice hilft dir gerne weiter:'),
|
|
'en': ('This form is temporarily unavailable.',
|
|
'Questions or want to book? Our customer service is happy to help:'),
|
|
}
|
|
|
|
CONTACT_NOTICE = '''<div class="static-contact-notice" style="background:#ebf3fa;border-left:4px solid #3d8ccb;padding:24px;margin:16px 0">
|
|
<p style="margin:0 0 8px;font-weight:700">{lead}</p>
|
|
<p style="margin:0 0 16px">{text}</p>
|
|
<p style="margin:0;display:flex;flex-wrap:wrap;gap:8px">
|
|
<a href="tel:{tel}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{phone}</a>
|
|
<a href="mailto:{email}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{email}</a>
|
|
</p>
|
|
</div>'''
|
|
|
|
|
|
def replace_elements(text, tag, start_re, replacement):
|
|
"""Replace each element whose start tag matches, including its balanced content. replacement
|
|
may be a function of the start tag match and the element, returning the new markup."""
|
|
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
|
|
out, pos, count = [], 0, 0
|
|
for m in start_re.finditer(text):
|
|
if m.start() < pos:
|
|
continue # nested inside an element already replaced
|
|
depth, end = 1, None
|
|
for t in token_re.finditer(text, m.end()):
|
|
depth += -1 if t.group(1) else 1
|
|
if depth == 0:
|
|
end = t.end()
|
|
break
|
|
if end is None:
|
|
continue # unbalanced markup, leave it alone
|
|
out += [text[pos:m.start()], replacement(m, text[m.start():end]) if callable(replacement) else replacement]
|
|
pos, count = end, count + 1
|
|
out.append(text[pos:])
|
|
return ''.join(out), count
|
|
|
|
URL_ATTR_RE = re.compile(r'(?:href|src|srcset|data-src|data-srcset|content|poster|action)="([^"]*)"')
|
|
CSS_URL_RE = re.compile(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)')
|
|
ASSET_EXT_RE = re.compile(r'\.(css|js|mjs|json|map|svg|png|jpe?g|gif|webp|avif|ico|woff2?|ttf|eot|otf|pdf|mp4|webm|txt|xml|webmanifest)$', re.I)
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""Report redirects instead of following them, so they become rewrite rules."""
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
OPENER = urllib.request.build_opener(NoRedirect)
|
|
|
|
|
|
def rewrite_rule(path, location, status):
|
|
pattern = re.escape(urllib.parse.unquote(path).strip('/')).replace('\\ ', '\\x20')
|
|
target = re.sub(r'([$%])', r'\\\1', location)
|
|
return '\tRewriteRule "^%s/?$" "%s" [R=%d,L,NE]' % (pattern, target, status)
|
|
|
|
|
|
def load_sites(only=None):
|
|
"""Read base and Development base variant from each site config."""
|
|
sites = {}
|
|
for name in sorted(os.listdir(SITES_DIR)):
|
|
path = os.path.join(SITES_DIR, name, 'config.yaml')
|
|
if not os.path.isfile(path) or (only and name not in only):
|
|
continue
|
|
with open(path, encoding='utf-8') as fh:
|
|
text = fh.read()
|
|
prod = re.search(r"^base:\s*'?([^'\s]+)", text, re.M)
|
|
dev = re.search(r"base:\s*'?(https://[^'\s]+\.ddev\.site)'?\s*\n\s*condition:\s*'applicationContext == \"Development\"'", text)
|
|
if prod and dev and prod.group(1).startswith('http'):
|
|
root = re.search(r'^rootPageId:\s*(\d+)', text, re.M)
|
|
sites[name] = {'prod': prod.group(1).rstrip('/'), 'dev': dev.group(1).rstrip('/'),
|
|
'root': int(root.group(1)) if root else 0, 'languages': site_languages(text)}
|
|
return sites
|
|
|
|
|
|
def site_languages(text):
|
|
"""languageId => base path prefix ('' for '/', '/en' for '/en/') of a site config."""
|
|
section = re.search(r'^languages:\n((?:[ \t-].*\n|\n)*)', text, re.M)
|
|
languages = {}
|
|
for item in re.split(r'^ -[ \t]*\n?', section.group(1) if section else '', flags=re.M):
|
|
language_id = re.search(r"^\s*languageId:\s*'?(\d+)", item, re.M)
|
|
base = re.search(r"^\s{4}base:\s*'?([^'\s]+)", item, re.M)
|
|
if language_id and base and base.group(1).startswith('/'):
|
|
languages[int(language_id.group(1))] = base.group(1).rstrip('/')
|
|
return languages
|
|
|
|
|
|
class Exporter:
|
|
def __init__(self, name, site, all_sites, out_dir, follow, workers, hybrid=False, live_ajax=False):
|
|
self.site = site
|
|
self.hybrid = hybrid
|
|
self.live_ajax = live_ajax
|
|
self.dynamic = dynamic_paths(site) if hybrid else set()
|
|
self.route_prefixes = set(route_pages(site).values()) if hybrid else set()
|
|
self.contact = CONTACTS.get(name, DEFAULT_CONTACT)
|
|
self.dev = site['dev']
|
|
self.dev_host = urllib.parse.urlsplit(self.dev).netloc
|
|
# Editors often paste absolute production URLs; those are internal links too
|
|
self.prod_host = urllib.parse.urlsplit(site['prod']).netloc
|
|
self.ajax_results = {}
|
|
self.ajax_locks = {}
|
|
self.out = out_dir
|
|
self.follow = follow
|
|
self.workers = workers
|
|
# dev → prod for every site, so cross-site links come out right too
|
|
self.host_map = [(s['dev'], s['prod']) for s in all_sites.values()]
|
|
# Some pages get their AJAX URLs below a parent slug (/reisen-fuer-gruppen/ajax/html?...)
|
|
self.ajax_attr_re = re.compile(r'([a-zA-Z][\w:.-]*)="((?:%s)?(?:/[\w-]+)*%s[^"]*)"' % (re.escape(self.dev), re.escape(AJAX_PATH)))
|
|
self.seen = set()
|
|
self.assets = set()
|
|
self.lock = threading.Lock()
|
|
self.errors = []
|
|
self.crawl_redirects = []
|
|
self.follow_redirects = follow
|
|
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'form_islands': 0, 'snowreports': 0, 'searchbars': 0, 'search': 0, 'dynamic': 0}
|
|
|
|
# --- helpers -----------------------------------------------------------
|
|
|
|
def fetch(self, url, extra_headers=None, log_errors=True):
|
|
"""Return status, body and (for redirects) the Location header."""
|
|
headers = {'User-Agent': 'ep-static-export'}
|
|
headers.update(extra_headers or {})
|
|
parts = urllib.parse.urlsplit(url)
|
|
if parts.scheme == 'https' and parts.hostname.endswith('.ddev.site'):
|
|
# Talk to the container's own web server, bypassing ddev-router (works for hostnames
|
|
# added without "ddev restart"); TYPO3 still sees HTTPS via the forwarded header
|
|
url = urllib.parse.urlunsplit(('http', '127.0.0.1') + tuple(parts[2:]))
|
|
headers.update({'Host': parts.hostname, 'X-Forwarded-Proto': 'https'})
|
|
req = urllib.request.Request(url, headers=headers)
|
|
try:
|
|
with OPENER.open(req, timeout=60) as res:
|
|
return res.status, res.read().decode('utf-8', 'replace'), None
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read().decode('utf-8', 'replace'), e.headers.get('Location')
|
|
except Exception as e: # noqa: BLE001 - report and continue
|
|
if log_errors:
|
|
self.error('%s: %s' % (url, e))
|
|
return 0, '', None
|
|
|
|
def error(self, msg):
|
|
with self.lock:
|
|
self.errors.append(msg)
|
|
|
|
def write(self, rel_path, content):
|
|
path = os.path.join(self.out, rel_path.lstrip('/'))
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, 'w', encoding='utf-8') as fh:
|
|
fh.write(content)
|
|
|
|
def to_prod(self, text):
|
|
for dev, prod in self.host_map:
|
|
text = text.replace(dev, prod)
|
|
text = text.replace(dev.replace('/', '\\/'), prod.replace('/', '\\/'))
|
|
text = text.replace(urllib.parse.quote(dev, safe=''), urllib.parse.quote(prod, safe=''))
|
|
return text
|
|
|
|
def local_path(self, url):
|
|
"""Path on this site for a URL, or None if it points elsewhere."""
|
|
parts = urllib.parse.urlsplit(html.unescape(url))
|
|
if parts.scheme and parts.scheme not in ('http', 'https'):
|
|
return None
|
|
if parts.netloc and parts.netloc not in (self.dev_host, self.prod_host) and parts.netloc not in CDN_HOSTS:
|
|
return None
|
|
if not parts.path.startswith('/') or parts.path.startswith('//'):
|
|
return None
|
|
return parts
|
|
|
|
def collect(self, text):
|
|
"""Collect asset paths and (if following) internal page links."""
|
|
links = set()
|
|
for raw in URL_ATTR_RE.findall(text) + CSS_URL_RE.findall(text):
|
|
for candidate in raw.split(','): # srcset
|
|
candidate = candidate.strip().split(' ')[0]
|
|
parts = self.local_path(candidate)
|
|
if not parts:
|
|
continue
|
|
path = urllib.parse.unquote(parts.path)
|
|
if ASSET_EXT_RE.search(path):
|
|
self.assets.add(path)
|
|
elif self.follow and not parts.query and parts.netloc in ('', self.dev_host, self.prod_host) \
|
|
and not path.startswith(('/typo3', AJAX_PATH)):
|
|
links.add(self.dev + path)
|
|
return links
|
|
|
|
# --- pages and AJAX tables ---------------------------------------------
|
|
|
|
def ajax_file(self, url):
|
|
"""Fetch an AJAX table, store it, return its static URL."""
|
|
url = html.unescape(url)
|
|
if url.startswith('/'):
|
|
url = self.dev + url
|
|
name = '/_ajax/%s.html' % hashlib.sha1(url.replace(self.dev, '').encode()).hexdigest()
|
|
with self.lock:
|
|
name_lock = self.ajax_locks.setdefault(name, threading.RLock())
|
|
# Every reference waits for the first fetch, so a failed table keeps its original URL everywhere
|
|
with name_lock:
|
|
if name not in self.ajax_results:
|
|
# Set before fetching: a response referencing itself must not recurse.
|
|
# The JS appends "&pa=..."; the dummy query keeps that harmless
|
|
self.ajax_results[name] = name + '?s=1'
|
|
status, body, _ = self.fetch(url)
|
|
if status != 200:
|
|
self.error('%s → %s' % (url, status))
|
|
self.ajax_results[name] = url
|
|
else:
|
|
self.write(name, self.to_prod(self.process(self.strip_dynamic(body, ''))))
|
|
with self.lock:
|
|
self.stats['ajax'] += 1
|
|
return self.ajax_results[name]
|
|
|
|
def strip_dynamic(self, text, path):
|
|
"""Swap contact forms for a contact notice and drop the search UI."""
|
|
phone, tel, email = self.contact
|
|
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
|
|
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
|
|
islands = snowreports = searchbars = 0
|
|
if self.hybrid and path:
|
|
text, searchbars = replace_elements(text, 'div', SEARCHBAR_RE,
|
|
lambda m, element: self.searchbar_island(path, m, element))
|
|
island = FORM_ISLAND.format(url=html.escape(path.rstrip('/') + '/form'), notice=notice)
|
|
text, islands = replace_elements(text, 'div', FORM_PLUGIN_RE, island)
|
|
text, snowreports = replace_elements(text, 'div', SNOWREPORT_BOX_RE,
|
|
lambda m, element: self.snowreport_island(text, path, m, element))
|
|
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
|
|
removed = 0
|
|
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
|
|
text, n = replace_elements(text, tag, start_re, '')
|
|
removed += n
|
|
with self.lock:
|
|
self.stats['forms'] += forms
|
|
self.stats['form_islands'] += islands
|
|
self.stats['snowreports'] += snowreports
|
|
self.stats['searchbars'] += searchbars
|
|
self.stats['search'] += removed
|
|
return text
|
|
|
|
def searchbar_island(self, path, m, element):
|
|
"""The exported search bar with the GET form of TYPO3 13, replaced on load by the
|
|
current bar from TYPO3 13 (open or collapsed, as exported)."""
|
|
element, forms = SEARCHBAR_FORM_RE.subn(
|
|
lambda f: '<form action="%s" method="get" data-searchbar-target="form">'
|
|
% html.escape(html.unescape(f.group(1)).split('?')[0]), element) # host: see process()
|
|
element, signed = SEARCHBAR_SIGNED_FIELDS_RE.subn('', element)
|
|
element, fields = SEARCHBAR_FIELD_RE.subn(r'name="\1"', element)
|
|
if (forms, signed, fields) != (1, 1, 6):
|
|
self.error('%s: search bar form not as expected (form %d, signed fields %d, fields %d), left out'
|
|
% (path, forms, signed, fields))
|
|
return ''
|
|
variant = 'searchbar-collapsed' if 'data-searchbar-collapsed-value="true"' in element else 'searchbar'
|
|
url = ('/en/' if path.startswith('/en/') else '/') + variant
|
|
start = m.group(0)
|
|
return start[:-1] + SEARCHBAR_ISLAND_ATTRIBUTES.format(url=url) + '>' + element[len(start):]
|
|
|
|
def snowreport_island(self, text, path, m, element):
|
|
"""The placeholder loading the snow report box from TYPO3 13, for the content element of
|
|
the anchor right before it; without one the box stays as exported."""
|
|
anchor = CONTENT_ANCHOR_RE.search(text, max(0, m.start() - 32), m.start())
|
|
if not anchor:
|
|
self.error('%s: snow report without content element anchor' % path)
|
|
return element
|
|
return SNOWREPORT_ISLAND.format(url=html.escape(path.rstrip('/') + '/snowreport'), uid=anchor.group(1))
|
|
|
|
def process(self, text):
|
|
"""Replace AJAX table URLs (recursively fetching them) and collect assets."""
|
|
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
|
|
'%s="%s"' % (m.group(1), html.escape(self.ajax_file(m.group(2)))), text)
|
|
self.collect(text)
|
|
return text
|
|
|
|
def is_live_ajax(self, url):
|
|
"""Hybrid --live-ajax: keep the URL when TYPO3 13 serves the action."""
|
|
params = dict(LIVE_AJAX_PARAM_RE.findall(html.unescape(url)))
|
|
return self.live_ajax and (params.get('controller'), params.get('action')) in LIVE_AJAX
|
|
|
|
def is_dynamic(self, path):
|
|
"""Hybrid: product pages (with their /detail/ variants) and search pages belong to TYPO3 13."""
|
|
path = path.rstrip('/') + '/'
|
|
return (path in self.dynamic or any(path.startswith(p + 'detail/') for p in self.dynamic)
|
|
or any(path.startswith(p) and path != p for p in self.route_prefixes))
|
|
|
|
def safe_page(self, url):
|
|
try:
|
|
return self.page(url)
|
|
except Exception as e: # noqa: BLE001 - one broken page must not stop the export
|
|
self.error('%s: %s: %s' % (url, type(e).__name__, e))
|
|
return set()
|
|
|
|
def page(self, url):
|
|
status, body, location = self.fetch(url)
|
|
if status in (301, 302, 303, 307, 308) and location:
|
|
location = urllib.parse.urljoin(url, location)
|
|
with self.lock:
|
|
self.crawl_redirects.append(rewrite_rule(urllib.parse.urlsplit(url).path, self.to_prod(location), status))
|
|
internal = location.startswith(self.dev + '/') and '?' not in location
|
|
return {location} if internal and self.follow_redirects else set()
|
|
if status != 200:
|
|
self.error('%s → %s' % (url, status))
|
|
return set()
|
|
path = urllib.parse.urlsplit(url).path
|
|
if self.hybrid and self.is_dynamic(path):
|
|
# Served by TYPO3 13 through index.php; only its links are followed
|
|
with self.lock:
|
|
self.stats['dynamic'] += 1
|
|
return self.collect(body)
|
|
if not self.hybrid and SEARCH_PAGE_RE.search(path):
|
|
# Search does not work statically: send visitors to the homepage instead
|
|
with self.lock:
|
|
self.crawl_redirects.append(rewrite_rule(path, self.site['prod'] + '/', 302))
|
|
return set()
|
|
# Extensionless paths become directories, so /a and /a/b/ can both exist
|
|
target = path if re.search(r'\.[a-z0-9]+$', path, re.I) else path.rstrip('/') + '/index.html'
|
|
body = self.strip_dynamic(body, path)
|
|
links = self.collect(body)
|
|
self.write(target, self.to_prod(self.process(body)))
|
|
with self.lock:
|
|
self.stats['pages'] += 1
|
|
return links
|
|
|
|
def sitemap(self):
|
|
"""Export sitemap index and children as plain files, return page URLs."""
|
|
status, index, _ = self.fetch(self.dev + '/sitemap.xml')
|
|
if status != 200:
|
|
print(' note: no sitemap.xml (%s), crawling from / only' % status)
|
|
return []
|
|
urls = []
|
|
for loc in re.findall(r'<loc>([^<]+)</loc>', index):
|
|
child_url = html.unescape(loc)
|
|
name = urllib.parse.parse_qs(urllib.parse.urlsplit(child_url).query).get('sitemap', ['pages'])[0]
|
|
_, child, _ = self.fetch(child_url)
|
|
urls += [html.unescape(u) for u in re.findall(r'<loc>([^<]+)</loc>', child)]
|
|
self.write('/sitemap-%s.xml' % name, self.to_prod(child))
|
|
index = index.replace(loc, self.dev + '/sitemap-%s.xml' % name)
|
|
self.write('/sitemap.xml', self.to_prod(index))
|
|
return [u for u in urls if u.startswith(self.dev)]
|
|
|
|
def crawl(self, start):
|
|
queue = list(dict.fromkeys(start))
|
|
self.seen.update(queue)
|
|
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
|
|
while queue:
|
|
batch, queue = queue, []
|
|
for links in pool.map(self.safe_page, batch):
|
|
for link in links:
|
|
if link not in self.seen:
|
|
self.seen.add(link)
|
|
queue.append(link)
|
|
|
|
def redirects(self):
|
|
"""Backend redirects (sys_redirect) as RewriteRules, targets resolved by TYPO3 itself."""
|
|
host = urllib.parse.urlsplit(self.site['prod']).netloc
|
|
where = "deleted=0 AND disabled=0 AND source_host IN ('%s', '*')" % host
|
|
try:
|
|
out = subprocess.run(
|
|
['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e',
|
|
'SELECT source_path FROM sys_redirect WHERE %s AND is_regexp=0; '
|
|
'SELECT COUNT(*) FROM sys_redirect WHERE %s AND is_regexp=1' % (where, where)],
|
|
capture_output=True, text=True, check=True).stdout.splitlines()
|
|
except (OSError, subprocess.CalledProcessError) as e:
|
|
self.error('sys_redirect: %s' % e)
|
|
return ''
|
|
paths, regexp = sorted(set(out[:-1])), int(out[-1])
|
|
skipped = ['regexp redirects (not exported): %d' % regexp] if regexp else []
|
|
|
|
def resolve(path):
|
|
if '?' in path:
|
|
return path, 0, 'query string in source path'
|
|
conn = http.client.HTTPConnection('127.0.0.1', 80, timeout=60)
|
|
try:
|
|
conn.request('GET', urllib.parse.quote(path, safe="/%:@!$&'()*+,;=-._~"), headers={'Host': host})
|
|
res = conn.getresponse()
|
|
res.read()
|
|
return path, res.status, res.getheader('Location')
|
|
finally:
|
|
conn.close()
|
|
|
|
rules = ['\t# Redirects from the TYPO3 backend (sys_redirect), resolved by TYPO3 at export time']
|
|
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
|
|
for path, status, location in pool.map(resolve, paths):
|
|
if status not in (301, 302, 303, 307, 308) or not location:
|
|
# Mostly dead entries (target page gone) that 404 in production already
|
|
skipped.append('%s → %s %s' % (path, status, location or ''))
|
|
continue
|
|
# Resolve like a browser: TYPO3 sends "https:/path" for relative targets
|
|
location = urllib.parse.urljoin(self.dev + path, location)
|
|
if location.rstrip('/') == (self.dev + path).rstrip('/'):
|
|
# TYPO3 matches "/a" for "/a/" too, so a redirect to the page at "/a/" loops
|
|
skipped.append('%s → %s (redirects to itself)' % (path, location))
|
|
continue
|
|
target = urllib.parse.urlsplit(location)
|
|
if target.netloc == self.dev_host and ASSET_EXT_RE.search(target.path):
|
|
self.assets.add(urllib.parse.unquote(target.path))
|
|
rules.append(rewrite_rule(path, self.to_prod(location), status))
|
|
self.stats['redirects'] = len(rules) - 1
|
|
self.stats['redirects_skipped'] = len(skipped)
|
|
with open(self.out.rstrip('/') + '-redirects-skipped.txt', 'w', encoding='utf-8') as fh:
|
|
fh.write('\n'.join(skipped) + '\n')
|
|
return '\n'.join(rules) + '\n'
|
|
|
|
def special_files(self):
|
|
status, body, _ = self.fetch(self.dev + '/robots.txt')
|
|
if status == 200:
|
|
self.write('/robots.txt', self.to_prod(body))
|
|
# TYPO3 error page, served by Apache via ErrorDocument 404
|
|
status, body, _ = self.fetch(self.dev + '/static-export-not-found/')
|
|
if status == 404 and body:
|
|
self.write('/404.html', self.to_prod(self.process(self.strip_dynamic(body, '/'))))
|
|
|
|
# --- static assets -----------------------------------------------------
|
|
|
|
def copy_assets(self):
|
|
pending, copied, missing = list(self.assets), set(), []
|
|
while pending:
|
|
path = pending.pop()
|
|
if path in copied:
|
|
continue
|
|
copied.add(path)
|
|
src = os.path.join(PUBLIC, path.lstrip('/'))
|
|
if not os.path.isfile(src):
|
|
missing.append(path)
|
|
continue
|
|
dst = os.path.join(self.out, path.lstrip('/'))
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
if path.endswith('.css'): # fonts and images referenced from CSS
|
|
with open(src, encoding='utf-8', errors='replace') as fh:
|
|
for ref in CSS_URL_RE.findall(fh.read()):
|
|
if ref.startswith(('data:', 'http', '#')):
|
|
continue
|
|
ref = urllib.parse.urlsplit(ref).path
|
|
pending.append(os.path.normpath(os.path.join(os.path.dirname(path), ref)))
|
|
return len(copied) - len(missing), missing
|
|
|
|
|
|
STATIC_HEADER = '''### Begin: Static export ###
|
|
# Generated by .ddev/static-export/export.py from public/.htaccess.
|
|
# No PHP: TYPO3's index.php catch-all and StaticFileCache rules are removed.
|
|
DirectoryIndex index.html
|
|
ErrorDocument 404 /404.html
|
|
Options -Indexes -MultiViews
|
|
<IfModule mod_php7.c>
|
|
php_flag engine off
|
|
</IfModule>
|
|
### End: Static export ###
|
|
|
|
'''
|
|
|
|
# --- TYPO3 backchannel API (/api/hotel, /api/product) ----------------------------------------
|
|
|
|
# MyE&P calls the API on this site; ApiMiddleware itself answers on every host
|
|
API_SITES = {'ep-reisen'}
|
|
API_CODE_RE = re.compile(r'^[A-Za-z0-9_-]+$')
|
|
|
|
|
|
def mysql_rows(sql):
|
|
out = subprocess.run(['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', sql],
|
|
capture_output=True, text=True, check=True).stdout
|
|
return [line.split('\t') for line in out.splitlines() if line]
|
|
|
|
|
|
# Records that only exist as URL arguments of one page (route enhancers), not linked anywhere:
|
|
# (page uid, table, field, pattern the value must match or None)
|
|
ROUTE_RECORDS = (
|
|
(1846, 'tx_epproducts_domain_model_product', 'path_segment', None), # /gruppe/{product}
|
|
(2736, 'tx_epproducts_domain_model_travelinfo', 'path_segment', None), # /reiseinformationen/{name}
|
|
(2736, 'tx_epproducts_domain_model_travelinfo', 'travel_code', r'[A-Z]+\d{6}'), # /reiseinformationen/{code}
|
|
)
|
|
|
|
|
|
def page_tree_check(site):
|
|
"""Return a function telling whether a default-language page uid belongs to the site."""
|
|
parents = {int(uid): int(pid) for uid, pid in mysql_rows('SELECT uid, pid FROM pages WHERE sys_language_uid = 0')}
|
|
|
|
def in_site(uid):
|
|
for _ in range(100):
|
|
if uid == site['root']:
|
|
return True
|
|
if uid not in parents:
|
|
return False
|
|
uid = parents[uid]
|
|
return False
|
|
return in_site
|
|
|
|
|
|
def route_pages(site):
|
|
"""Slug (with trailing slash) of each route page of ROUTE_RECORDS on this site, by page uid."""
|
|
in_site = page_tree_check(site)
|
|
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
|
|
pages = {}
|
|
for uid in sorted({r[0] for r in ROUTE_RECORDS}):
|
|
rows = mysql_rows('SELECT slug FROM pages WHERE uid = %d AND deleted = 0 AND hidden = 0' % uid)
|
|
if rows and in_site(uid):
|
|
pages[uid] = base + rows[0][0].rstrip('/') + '/'
|
|
return pages
|
|
|
|
|
|
def route_urls(site):
|
|
"""Start URLs for the records of ROUTE_RECORDS (not reachable through links or the sitemap)."""
|
|
pages = route_pages(site)
|
|
urls = set()
|
|
for uid, table, field, pattern in ROUTE_RECORDS:
|
|
if uid not in pages:
|
|
continue
|
|
for (value,) in mysql_rows("SELECT DISTINCT %s FROM %s WHERE deleted = 0 AND hidden = 0 AND %s <> ''" % (field, table, field)):
|
|
if pattern is None or re.fullmatch(pattern, value):
|
|
urls.add(site['dev'].rstrip('/') + pages[uid] + urllib.parse.quote(value) + '/')
|
|
return sorted(urls)
|
|
|
|
|
|
def dynamic_paths(site):
|
|
"""Paths (with trailing slash) of the pages TYPO3 13 serves in hybrid mode: pages of this site
|
|
holding the product detail or search result plugin, in every language of the site."""
|
|
in_site = page_tree_check(site)
|
|
rows = mysql_rows(
|
|
"SELECT DISTINCT p.slug, p.sys_language_uid, IF(p.sys_language_uid = 0, p.uid, p.l10n_parent) "
|
|
"FROM pages p JOIN tt_content c ON c.pid = p.uid "
|
|
"WHERE p.deleted = 0 AND p.hidden = 0 AND c.deleted = 0 AND c.hidden = 0 AND c.CType = 'list' "
|
|
"AND c.list_type IN ('epproducts_product_detail', 'epproducts_searchresult')")
|
|
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
|
|
paths = set()
|
|
for slug, language, default_uid in rows:
|
|
if int(language) in site['languages'] and in_site(int(default_uid)):
|
|
paths.add(base + site['languages'][int(language)] + slug.rstrip('/') + '/')
|
|
return paths
|
|
|
|
|
|
FRONT_CONTROLLER_STUB = """<?php
|
|
// Generated by .ddev/static-export/export.py --hybrid: everything not in this static export is
|
|
// handled by TYPO3 13 (product pages, /gruppe/, travel info, search, /api/*, /ajax/*, 404).
|
|
// readlink() is not cached, unlike the realpath of a symlinked include path, so right after a
|
|
// deploy every request already loads the new release (under new paths, so OPcache needs no reset).
|
|
$current = %r;
|
|
$release = readlink($current);
|
|
if ($release === false) {
|
|
http_response_code(503);
|
|
exit;
|
|
}
|
|
$release = $release[0] === '/' ? $release : dirname($current) . '/' . $release;
|
|
|
|
// Public extension resources (/_assets/<hash>/...) come from the release as well: Apache does not
|
|
// follow a symlink from the docroot into the release on every vhost, PHP does.
|
|
$path = rawurldecode((string)strtok($_SERVER['REQUEST_URI'] ?? '', '?'));
|
|
if (str_starts_with($path, '/_assets/')) {
|
|
$types = [
|
|
'css' => 'text/css; charset=utf-8', 'js' => 'text/javascript; charset=utf-8', 'json' => 'application/json',
|
|
'map' => 'application/json', 'svg' => 'image/svg+xml', 'png' => 'image/png', 'jpg' => 'image/jpeg',
|
|
'jpeg' => 'image/jpeg', 'gif' => 'image/gif', 'webp' => 'image/webp', 'avif' => 'image/avif',
|
|
'ico' => 'image/x-icon', 'woff' => 'font/woff', 'woff2' => 'font/woff2', 'ttf' => 'font/ttf',
|
|
'otf' => 'font/otf', 'eot' => 'application/vnd.ms-fontobject',
|
|
];
|
|
$file = realpath($release . '/public' . $path);
|
|
$type = $types[strtolower(pathinfo((string)$file, PATHINFO_EXTENSION))] ?? null;
|
|
// realpath() resolves the publishing symlinks; anything outside the release is refused
|
|
if ($file === false || $type === null || !is_file($file) || !str_starts_with($file, realpath($release) . '/')) {
|
|
http_response_code(404);
|
|
exit;
|
|
}
|
|
header('Content-Type: ' . $type);
|
|
header('Content-Length: ' . filesize($file));
|
|
header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($file)) . ' GMT');
|
|
// Content-hashed build files never change under their name (as in the .htaccess for static files).
|
|
// The Expires header keeps mod_expires from adding a second, conflicting max-age.
|
|
$maxAge = preg_match('/\\.[0-9a-f]{8}\\.\\w+$/', $file) ? 31536000 : 86400;
|
|
header('Cache-Control: public, max-age=' . $maxAge . ($maxAge === 31536000 ? ', immutable' : ''));
|
|
header('Expires: ' . gmdate('D, d M Y H:i:s', time() + $maxAge) . ' GMT');
|
|
readfile($file);
|
|
exit;
|
|
}
|
|
|
|
require $release . '/public/index.php';
|
|
"""
|
|
|
|
|
|
def cms_api_key():
|
|
php = '$c = include "%s/public/typo3conf/LocalConfiguration.php"; echo $c["EXTENSIONS"]["ep_products"]["cmsApiKey"] ?? "";' % APP_ROOT
|
|
key = subprocess.run(['php', '-r', php], capture_output=True, text=True, check=True).stdout.strip()
|
|
if not API_CODE_RE.match(key):
|
|
sys.exit('cmsApiKey missing or with characters the .htaccess rules cannot match literally')
|
|
return key
|
|
|
|
|
|
class ApiExporter:
|
|
"""Snapshot /api/hotel and /api/product as JSON files, answered by TYPO3 itself.
|
|
|
|
TYPO3 data is frozen while nobody edits, so the responses are too. A product+hotel response is
|
|
the product response with "hotel": <hotel response> appended (ApiMiddleware json-encodes one
|
|
array), so pair files are assembled from the single responses and a sample is checked against
|
|
TYPO3. Pairs that were not generated get the response TYPO3 gives for an unknown hotel.
|
|
"""
|
|
|
|
def __init__(self, exp):
|
|
self.exp = exp
|
|
self.key = cms_api_key()
|
|
self.stats = {'hotels': 0, 'products': 0, 'pairs': 0, 'skipped_codes': []}
|
|
|
|
def get(self, query):
|
|
url = self.exp.dev + '/api/' + query
|
|
for _ in range(3): # the first call of a record can be slow while TYPO3 renders its images
|
|
status, body, _ = self.exp.fetch(url, {'X-Api-Key': self.key}, log_errors=False)
|
|
if status:
|
|
return status, body
|
|
self.exp.error('api: %s failed 3 times' % query)
|
|
return status, body
|
|
|
|
def responses(self, endpoint, codes):
|
|
"""code -> body for every code TYPO3 answers with 200."""
|
|
codes = sorted(set(codes))
|
|
fetch = lambda c: self.get('%s?%s=%s' % (endpoint, endpoint, urllib.parse.quote(c)))
|
|
with concurrent.futures.ThreadPoolExecutor(self.exp.workers) as pool:
|
|
# Warm-up pass: responses change while TYPO3 is still generating resized images
|
|
list(pool.map(fetch, codes))
|
|
results = pool.map(fetch, codes)
|
|
return {c: body for c, (status, body) in zip(codes, results) if status == 200}
|
|
|
|
def write(self, path, body):
|
|
self.exp.write('/_api/' + path, self.exp.to_prod(body))
|
|
|
|
def run(self):
|
|
hotel_codes = collections.defaultdict(set)
|
|
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_hotel WHERE deleted=0 AND code<>''"):
|
|
hotel_codes[uid].add(code)
|
|
aliases = mysql_rows("SELECT entity, code FROM tx_epproducts_domain_model_matchcode WHERE deleted=0 AND code<>''")
|
|
for uid, code in aliases:
|
|
hotel_codes[uid].add(code)
|
|
product_codes = collections.defaultdict(set)
|
|
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_product WHERE deleted=0 AND code<>''"):
|
|
product_codes[uid].add(code)
|
|
pairs = mysql_rows(
|
|
"SELECT uid_local, uid_foreign FROM tx_epproducts_product_hotel_mm"
|
|
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_date WHERE deleted=0"
|
|
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_hotel_mapping WHERE deleted=0 AND hidden=0")
|
|
|
|
def safe(codes):
|
|
ok = {c for c in codes if API_CODE_RE.match(c)}
|
|
self.stats['skipped_codes'] += sorted(set(codes) - ok)
|
|
return ok
|
|
|
|
# ApiMiddleware matches matchcode aliases for products too (join on matchcode.entity)
|
|
hotels = self.responses('hotel', safe({c for cs in hotel_codes.values() for c in cs}))
|
|
products = self.responses('product', safe({c for cs in product_codes.values() for c in cs} | {c for _, c in aliases}))
|
|
# An empty code is not "missing" for ApiMiddleware: it matches a record with an empty code
|
|
for endpoint, found in (('hotel', hotels), ('product', products)):
|
|
status, body = self.get('%s?%s=' % (endpoint, endpoint))
|
|
if status == 200:
|
|
found['@empty'] = body
|
|
for code, body in hotels.items():
|
|
self.write('hotel/%s.json' % code, body)
|
|
for code, body in products.items():
|
|
self.write('product/%s.json' % code, body)
|
|
self.write('product/%s/_unknown-hotel.json' % code, body[:-1] + ',"hotel":[]}')
|
|
# Product response without its closing brace, for pairs assembled by SSI (pair.shtml)
|
|
self.write('product-part/%s.part' % code, body[:-1])
|
|
self.exp.write('/_api/pair.shtml', '<!--#include virtual="/_api/product-part/${REDIRECT_API_PRODUCT}.part" -->'
|
|
',"hotel":<!--#include virtual="/_api/hotel/${REDIRECT_API_HOTEL}.json" -->}')
|
|
written = set()
|
|
for product_uid, hotel_uid in pairs:
|
|
for pc in product_codes.get(product_uid, ()):
|
|
for hc in hotel_codes.get(hotel_uid, ()):
|
|
if pc in products and hc in hotels and (pc, hc) not in written:
|
|
written.add((pc, hc))
|
|
self.write('product/%s/%s.json' % (pc, hc), products[pc][:-1] + ',"hotel":' + hotels[hc] + '}')
|
|
self.stats.update(hotels=len(hotels), products=len(products), pairs=len(written))
|
|
|
|
# The assembled files must be byte-identical to what TYPO3 answers
|
|
sample = sorted(written)[::max(1, len(written) // 40)] + [(pc, 'NO-SUCH-HOTEL') for pc in sorted(p for p in products if p != '@empty')[:5]]
|
|
for pc, hc in sample:
|
|
status, body = self.get('product?product=%s&hotel=%s' % (pc, urllib.parse.quote(hc)))
|
|
expected = products[pc][:-1] + (',"hotel":' + hotels[hc] if hc in hotels else ',"hotel":[]') + '}'
|
|
if status != 200 or body != expected:
|
|
self.exp.error('api: assembled /api/product?product=%s&hotel=%s differs from TYPO3' % (pc, hc))
|
|
|
|
# Error bodies exactly as ApiMiddleware sends them
|
|
for name, query in (('hotel-400', 'hotel'), ('hotel-404', 'hotel?hotel=NO-SUCH-CODE'),
|
|
('product-400', 'product'), ('product-404', 'product?product=NO-SUCH-CODE')):
|
|
self.write('error/%s.json' % name, self.get(query)[1])
|
|
self.write('error/403.json', self.exp.fetch(self.exp.dev + '/api/hotel?hotel=x')[1])
|
|
return API_RULES.replace('@KEY@', self.key)
|
|
|
|
|
|
API_HEADER = r'''# TYPO3 backchannel API: error bodies per endpoint, as ApiMiddleware sends them
|
|
AddCharset utf-8 .json
|
|
<If "%{REQUEST_URI} == '/api/hotel'">
|
|
ErrorDocument 400 /_api/error/hotel-400.json
|
|
ErrorDocument 403 /_api/error/403.json
|
|
ErrorDocument 404 /_api/error/hotel-404.json
|
|
</If>
|
|
<If "%{REQUEST_URI} == '/api/product'">
|
|
ErrorDocument 400 /_api/error/product-400.json
|
|
ErrorDocument 403 /_api/error/403.json
|
|
ErrorDocument 404 /_api/error/product-404.json
|
|
</If>
|
|
# Product + hotel pairs that were not pre-generated are assembled by SSI from the single responses
|
|
<IfModule mod_include.c>
|
|
<Files "pair.shtml">
|
|
Options +Includes
|
|
SetOutputFilter INCLUDES
|
|
ForceType "application/json; charset=utf-8"
|
|
</Files>
|
|
</IfModule>
|
|
|
|
'''
|
|
|
|
API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JSON snapshots taken from
|
|
# TYPO3 at export time; same status codes and bodies as ApiMiddleware
|
|
RewriteCond %{ENV:REDIRECT_STATUS} ^$
|
|
RewriteCond %{IS_SUBREQ} false
|
|
RewriteRule ^_api/ - [R=404,L]
|
|
RewriteCond %{HTTP:X-Api-Key} !=@KEY@
|
|
RewriteCond %{QUERY_STRING} !(^|&)key=@KEY@(&|$)
|
|
RewriteRule ^api/(hotel|product)$ - [R=403,L]
|
|
# Only file-safe codes are taken over; anything else leaves the variable unset (404 / "hotel": [])
|
|
RewriteCond %{QUERY_STRING} (^|&)hotel=([A-Za-z0-9_-]+)(&|$)
|
|
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:%2]
|
|
RewriteCond %{QUERY_STRING} (^|&)product=([A-Za-z0-9_-]+)(&|$)
|
|
RewriteRule ^api/product$ - [E=API_PRODUCT:%2]
|
|
# An empty code matches a record with an empty code in TYPO3: snapshot file "@empty" (never a real code)
|
|
RewriteCond %{QUERY_STRING} (^|&)hotel=(&|$)
|
|
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:@empty]
|
|
RewriteCond %{QUERY_STRING} (^|&)product=(&|$)
|
|
RewriteRule ^api/product$ - [E=API_PRODUCT:@empty]
|
|
RewriteCond %{QUERY_STRING} !(^|&)hotel=
|
|
RewriteRule ^api/hotel$ - [R=400,L]
|
|
RewriteCond %{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)$
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
|
|
RewriteRule ^api/hotel$ _api/hotel/%{ENV:API_HOTEL}.json [L]
|
|
RewriteRule ^api/hotel$ - [R=404,L]
|
|
RewriteCond %{QUERY_STRING} !(^|&)product=
|
|
RewriteRule ^api/product$ - [R=400,L]
|
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
|
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json -f
|
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json [L]
|
|
# Pair not pre-generated but both known: assembled by SSI (exactly what TYPO3 returns)
|
|
<IfModule mod_include.c>
|
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
|
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/product-part/%{ENV:API_PRODUCT}.part -f
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
|
|
RewriteRule ^api/product$ _api/pair.shtml [L]
|
|
</IfModule>
|
|
# Hotel unknown (or no mod_include): TYPO3 answers with "hotel": []
|
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
|
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json -f
|
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json [L]
|
|
RewriteCond %{QUERY_STRING} !(^|&)hotel=
|
|
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
|
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}.json -f
|
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}.json [L]
|
|
RewriteRule ^api/product$ - [R=404,L]
|
|
'''
|
|
|
|
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
|
|
|
|
STATIC_REWRITE = r'''
|
|
# No PHP is ever served, also not via a symlinked fileadmin
|
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
|
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
|
|
# Search does not work statically: any search page goes to the homepage
|
|
RewriteRule (^|/)suche/?$ / [R=302,L]
|
|
'''
|
|
|
|
|
|
HYBRID_HEADER = '''### Begin: Static export (hybrid) ###
|
|
# Generated by .ddev/static-export/export.py --hybrid from public/.htaccess.
|
|
# Files of the static export win; anything else goes to TYPO3 13 through index.php.
|
|
DirectoryIndex index.html
|
|
ErrorDocument 404 /404.html
|
|
Options -Indexes -MultiViews
|
|
### End: Static export (hybrid) ###
|
|
|
|
'''
|
|
|
|
HYBRID_REWRITE = r'''
|
|
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
|
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
|
RewriteCond %{ENV:REDIRECT_STATUS} ^$ [OR]
|
|
RewriteCond %{REQUEST_URI} !^/index\.php$
|
|
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
|
|
# TYPO3 13 backend and install tool stay closed to the public
|
|
RewriteRule ^typo3(/|$) - [F,L]
|
|
'''
|
|
|
|
HYBRID_CATCH_ALL = ''' # Not part of the static export (a directory without index.html counts as missing) => TYPO3 13
|
|
RewriteCond %{REQUEST_FILENAME} !-f
|
|
RewriteCond %{REQUEST_FILENAME}/index.html !-f
|
|
RewriteRule ^.*$ %{ENV:CWD}index.php [QSA,L]
|
|
'''
|
|
|
|
|
|
def build_htaccess(hybrid=False):
|
|
"""TYPO3's .htaccess without the parts that hand requests to PHP (hybrid: with the hand-over)."""
|
|
with open(os.path.join(PUBLIC, '.htaccess'), encoding='utf-8') as fh:
|
|
text = fh.read()
|
|
text, sfc = re.subn(r'[ \t]*### Begin: StaticFileCache \(preparation\).*?### End: StaticFileCache ###\n', '', text, flags=re.S)
|
|
text, catch_all = re.subn(r'(?:[ \t]*#[^\n]*\n)*(?:[ \t]*RewriteCond %\{REQUEST_FILENAME\} !-[fdl]\n){3}[ \t]*RewriteRule \^\.\*\$ %\{ENV:CWD\}index\.php \[QSA,L\]\n',
|
|
lambda m: HYBRID_CATCH_ALL if hybrid else '', text)
|
|
rewrite = HYBRID_REWRITE if hybrid else STATIC_REWRITE
|
|
text, engine = re.subn(r'(\n[ \t]*RewriteEngine On\n)', r'\1' + (rewrite + REDIRECTS_PLACEHOLDER).replace('\\', '\\\\'), text, count=1)
|
|
leftover = re.sub(r'#[^\n]*', '', text.replace(HYBRID_CATCH_ALL, '') if hybrid else text)
|
|
if (sfc, catch_all, engine) != (1, 1, 1) or re.search(r'RewriteRule[^\n]*index\.php', leftover):
|
|
sys.exit('public/.htaccess changed shape, adjust build_htaccess() (sfc=%d, catch-all=%d, engine=%d)' % (sfc, catch_all, engine))
|
|
return (HYBRID_HEADER if hybrid else STATIC_HEADER) + text
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument('--site', action='append', help='site identifier from config/sites (repeatable, default: all)')
|
|
ap.add_argument('--url', action='append', help='export only these URLs (dev URLs) instead of the sitemap')
|
|
ap.add_argument('--no-follow', action='store_true', help='do not follow internal links beyond the start URLs')
|
|
ap.add_argument('--out', default=os.path.join(APP_ROOT, 'var', 'static-export'))
|
|
ap.add_argument('--workers', type=int, default=8)
|
|
ap.add_argument('--hybrid', action='store_true', help='leave product pages and search to TYPO3 13 (see module doc)')
|
|
ap.add_argument('--typo3-current', help='hybrid: absolute path of the TYPO3 13 "current" release symlink on the server')
|
|
ap.add_argument('--live-ajax', action='store_true', help='hybrid: keep AJAX URLs that TYPO3 13 serves instead of pre-fetching them')
|
|
args = ap.parse_args()
|
|
if args.hybrid and not args.typo3_current:
|
|
ap.error('--hybrid needs --typo3-current')
|
|
sys.stdout.reconfigure(line_buffering=True)
|
|
|
|
all_sites = load_sites()
|
|
selected = {k: v for k, v in all_sites.items() if not args.site or k in args.site}
|
|
if not selected:
|
|
sys.exit('No matching site in %s' % SITES_DIR)
|
|
|
|
htaccess = build_htaccess(args.hybrid)
|
|
failed = False
|
|
for name, site in selected.items():
|
|
out = os.path.join(args.out, name)
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
exp = Exporter(name, site, all_sites, out, follow=not args.no_follow and not args.url, workers=args.workers,
|
|
hybrid=args.hybrid, live_ajax=args.live_ajax)
|
|
print('== %s: %s → %s' % (name, site['dev'], site['prod']))
|
|
if args.url:
|
|
start = [u for u in args.url if u.startswith(site['dev'])]
|
|
else:
|
|
start = [site['dev'] + '/'] + exp.sitemap()
|
|
if not args.hybrid:
|
|
# Linked from nowhere (handed out to groups and customers); in hybrid mode TYPO3 13 serves them
|
|
routes = route_urls(site)
|
|
start += routes
|
|
if routes:
|
|
print(' route URLs (/gruppe/, /reiseinformationen/): %d' % len(routes))
|
|
exp.special_files()
|
|
exp.crawl(start)
|
|
rules = exp.redirects()
|
|
site_htaccess = htaccess
|
|
if args.hybrid:
|
|
# TYPO3 13 behind index.php
|
|
# /_assets/ is served by the stub too (no symlink: not every vhost follows one)
|
|
exp.write('/index.php', FRONT_CONTROLLER_STUB % args.typo3_current.rstrip('/'))
|
|
if name in API_SITES and not args.url and not args.hybrid:
|
|
api = ApiExporter(exp)
|
|
rules = api.run() + rules
|
|
site_htaccess = API_HEADER + htaccess
|
|
print(' api: %d hotels, %d products, %d product/hotel pairs, codes skipped (not file-safe): %s'
|
|
% (api.stats['hotels'], api.stats['products'], api.stats['pairs'], ', '.join(api.stats['skipped_codes']) or '-'))
|
|
if exp.crawl_redirects:
|
|
rules += '\t# Redirects returned by TYPO3 while crawling\n' + '\n'.join(sorted(exp.crawl_redirects)) + '\n'
|
|
exp.write('/.htaccess', site_htaccess.replace(REDIRECTS_PLACEHOLDER, '\n' + rules))
|
|
copied, missing = exp.copy_assets()
|
|
print(' pages: %d, ajax tables: %d, redirects: %d (%d skipped, see %s-redirects-skipped.txt), assets: %d copied, %d missing'
|
|
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
|
|
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
|
|
if args.hybrid:
|
|
print(' left to TYPO3 13: %d pages, forms loaded from TYPO3 13: %d, snow reports loaded from TYPO3 13: %d,'
|
|
' search bars refreshed from TYPO3 13: %d'
|
|
% (exp.stats['dynamic'], exp.stats['form_islands'], exp.stats['snowreports'], exp.stats['searchbars']))
|
|
for path in sorted(missing)[:20]:
|
|
print(' missing asset: %s' % path)
|
|
for msg in exp.errors[:20]:
|
|
print(' error: %s' % msg)
|
|
if len(exp.errors) > 20:
|
|
print(' ... %d more errors' % (len(exp.errors) - 20))
|
|
failed = failed or bool(exp.errors)
|
|
sys.exit(1 if failed else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|