feat: hybrid export loads EXT:form forms from TYPO3 13, multipart

allowed
This commit is contained in:
fromme committed 2026-10-22 16:26:15 +02:00
1 parent f4f0857870
commit 85b1bd1578
3 files changed
+99 -17

No files matched your search

+98 -17
View File
@@ -8,7 +8,7 @@ plain document roots that run without PHP.
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
and the search are left out (TYPO3 13 serves them through index.php), the search and the search are left out (TYPO3 13 serves them through index.php), the search
UI stays (except the search bar), and the API is answered live. UI stays (the search bar is refreshed from TYPO3 13), and the API is answered live.
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR] Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
[--hybrid --typo3-current PATH [--live-ajax]] [--hybrid --typo3-current PATH [--live-ajax]]
@@ -45,6 +45,46 @@ AJAX_PATH = '/ajax/'
# cannot work statically and are replaced by a contact notice # cannot work statically and are replaced by a contact notice
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I) CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
# Hybrid: EXT:form plugins (ep_theme Templates/Form/Render.html: <div class="py-8"><form ...>) are
# loaded from TYPO3 13 instead, from <page>/form (page type 1704, renders the page's form only).
# htmx swaps the placeholder on load and the form posts back there. ?ref= (the page a visitor came
# from) is passed along for the enquiry e-mails. Until the form arrives the placeholder shows the
# spinner of the form islands (ep_theme Partials/Form/Spinner.html, here in the text colour: the
# theme colours come with the island). The contact notice is shown without JavaScript and, taken
# from the noscript element, when TYPO3 13 does not answer.
FORM_PLUGIN_RE = re.compile(r'<div class="py-8">(?=\s*<form\b[^>]*\baction="[^"]*tx_form_formframework)', re.I)
FORM_ISLAND_FALLBACK = "this.removeAttribute('style'); this.innerHTML = this.querySelector('noscript').textContent"
FORM_SPINNER = (
'<svg style="width:4rem;height:4rem;opacity:.5" viewBox="0 0 44 44" xmlns="http://www.w3.org/2000/svg" stroke="currentColor">'
'<g fill="none" fill-rule="evenodd" stroke-width="2">'
+ ''.join('<circle cx="22" cy="22" r="1">'
'<animate attributeName="r" begin="%s" dur="1.8s" values="1; 20" calcMode="spline" keyTimes="0; 1" keySplines="0.165, 0.84, 0.44, 1" repeatCount="indefinite" />'
'<animate attributeName="stroke-opacity" begin="%s" dur="1.8s" values="1; 0" calcMode="spline" keyTimes="0; 1" keySplines="0.3, 0.61, 0.355, 1" repeatCount="indefinite" />'
'</circle>' % (begin, begin) for begin in ('0s', '-0.9s'))
+ '</g></svg>')
FORM_ISLAND = ('<div class="form-island" hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
' hx-vals=\'js:{{ref: new URLSearchParams(location.search).get("ref") || ""}}\''
' hx-on:htmx:response-error="' + FORM_ISLAND_FALLBACK + '"'
' hx-on:htmx:send-error="' + FORM_ISLAND_FALLBACK + '"'
' style="min-height:12rem;display:flex;align-items:center;justify-content:center">'
+ FORM_SPINNER + '<noscript>{notice}</noscript></div>')
# Hybrid: snow report lists (ep_theme Templates/Snowreport/List.html: a column of rows starting
# with flag and region, in the spacing wrapper of ep:container if the element has spacing) are
# loaded from TYPO3 13, from <page>/snowreport (page type 1705, renders the page's snow reports
# only, each in id="snowreport-<content uid>" around the spacing wrapper). The export keeps an
# empty placeholder only: no outdated snow depths, and nothing without JavaScript. When TYPO3 13
# does not answer or no longer has that list, the placeholder is removed.
SNOWREPORT_BOX_RE = re.compile(
r'(?<=</a>)\s*<div class="[^"]*">'
r'(?=(?:\s*<div class="flex flex-col space-y-2">)?\s*<div class="grid md:grid-cols-3 md:gap-y-0">'
r'\s*<div class="md:col-span-1 bg-ep-primary-dark p-2 flex items-start space-x-2">)', re.I)
CONTENT_ANCHOR_RE = re.compile(r'<a id="c(\d+)"></a>$')
SNOWREPORT_ISLAND = ('<div hx-get="{url}" hx-trigger="load" hx-select="#snowreport-{uid}" hx-swap="outerHTML"'
' hx-on:htmx:before-swap="event.detail.shouldSwap = event.detail.xhr.responseText'
'.includes(\'id=&quot;snowreport-{uid}&quot;\'); event.detail.shouldSwap || this.remove()"'
' hx-on:htmx:send-error="this.remove()"></div>')
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them, # Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
# and links to the search page (menu icon, "Alle anzeigen" with preset filters) # and links to the search page (menu icon, "Alle anzeigen" with preset filters)
SEARCH_ELEMENTS = ( SEARCH_ELEMENTS = (
@@ -55,12 +95,19 @@ SEARCH_ELEMENTS = (
) )
SEARCH_PAGE_RE = re.compile(r'/suche/$') SEARCH_PAGE_RE = re.compile(r'/suche/$')
# Hybrid: only the search bar goes (TYPO3 13 has no search bar plugin); links, menu icons and # Hybrid: the search UI stays, the search itself is served by TYPO3 13. The search bar
# the mobile filter overlay stay, the search itself is served by TYPO3 13 # (ep_theme Templates/Searchbar/Index.html) is replaced on load by the current one from TYPO3 13,
HYBRID_SEARCH_ELEMENTS = ( # from <language root>searchbar or searchbar-collapsed (page types 1706/1707, the same on every
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')), # page). Until then, and when TYPO3 13 does not answer, the exported bar stays: its form is
('button', re.compile(r'<button\b[^>]*\bdata-controller="searchbar-toggle"[^>]*>')), # rewritten to the GET form of TYPO3 13 (fields as SearchController::SEARCHBAR_PARAMETERS),
) # since TYPO3 13 has no processSearch and would reject the signed fields of TYPO3 10.
HYBRID_SEARCH_ELEMENTS = ()
# The whole start tag: its data-action contains "->"
SEARCHBAR_RE = re.compile(r'<div\b(?:[^>"]|"[^"]*")*?\sdata-controller="searchbar"(?:[^>"]|"[^"]*")*>')
SEARCHBAR_FORM_RE = re.compile(r'<form data-searchbar-target="form" method="post" name="searchParams" action="([^"]*)">')
SEARCHBAR_SIGNED_FIELDS_RE = re.compile(r'<div>(?:\s*<input type="hidden" name="tx_epproducts_searchresult\[__[^>]*>)+\s*</div>')
SEARCHBAR_FIELD_RE = re.compile(r'name="tx_epproducts_searchresult\[searchParams\]\[(\w+)\]"')
SEARCHBAR_ISLAND_ATTRIBUTES = ' hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on # Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl # 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash # argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
@@ -103,7 +150,8 @@ CONTACT_NOTICE = '''<div class="static-contact-notice" style="background:#ebf3fa
def replace_elements(text, tag, start_re, replacement): def replace_elements(text, tag, start_re, replacement):
"""Replace each element whose start tag matches, including its balanced content.""" """Replace each element whose start tag matches, including its balanced content. replacement
may be a function of the start tag match and the element, returning the new markup."""
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I) token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
out, pos, count = [], 0, 0 out, pos, count = [], 0, 0
for m in start_re.finditer(text): for m in start_re.finditer(text):
@@ -117,7 +165,7 @@ def replace_elements(text, tag, start_re, replacement):
break break
if end is None: if end is None:
continue # unbalanced markup, leave it alone continue # unbalanced markup, leave it alone
out += [text[pos:m.start()], replacement] out += [text[pos:m.start()], replacement(m, text[m.start():end]) if callable(replacement) else replacement]
pos, count = end, count + 1 pos, count = end, count + 1
out.append(text[pos:]) out.append(text[pos:])
return ''.join(out), count return ''.join(out), count
@@ -200,7 +248,7 @@ class Exporter:
self.errors = [] self.errors = []
self.crawl_redirects = [] self.crawl_redirects = []
self.follow_redirects = follow self.follow_redirects = follow
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'search': 0, 'dynamic': 0} self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'form_islands': 0, 'snowreports': 0, 'searchbars': 0, 'search': 0, 'dynamic': 0}
# --- helpers ----------------------------------------------------------- # --- helpers -----------------------------------------------------------
@@ -301,6 +349,14 @@ class Exporter:
phone, tel, email = self.contact phone, tel, email = self.contact
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de'] lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email) notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
islands = snowreports = searchbars = 0
if self.hybrid and path:
text, searchbars = replace_elements(text, 'div', SEARCHBAR_RE,
lambda m, element: self.searchbar_island(path, m, element))
island = FORM_ISLAND.format(url=html.escape(path.rstrip('/') + '/form'), notice=notice)
text, islands = replace_elements(text, 'div', FORM_PLUGIN_RE, island)
text, snowreports = replace_elements(text, 'div', SNOWREPORT_BOX_RE,
lambda m, element: self.snowreport_island(text, path, m, element))
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice) text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
removed = 0 removed = 0
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS): for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
@@ -308,9 +364,38 @@ class Exporter:
removed += n removed += n
with self.lock: with self.lock:
self.stats['forms'] += forms self.stats['forms'] += forms
self.stats['form_islands'] += islands
self.stats['snowreports'] += snowreports
self.stats['searchbars'] += searchbars
self.stats['search'] += removed self.stats['search'] += removed
return text return text
def searchbar_island(self, path, m, element):
"""The exported search bar with the GET form of TYPO3 13, replaced on load by the
current bar from TYPO3 13 (open or collapsed, as exported)."""
element, forms = SEARCHBAR_FORM_RE.subn(
lambda f: '<form action="%s" method="get" data-searchbar-target="form">'
% html.escape(html.unescape(f.group(1)).split('?')[0]), element) # host: see process()
element, signed = SEARCHBAR_SIGNED_FIELDS_RE.subn('', element)
element, fields = SEARCHBAR_FIELD_RE.subn(r'name="\1"', element)
if (forms, signed, fields) != (1, 1, 6):
self.error('%s: search bar form not as expected (form %d, signed fields %d, fields %d), left out'
% (path, forms, signed, fields))
return ''
variant = 'searchbar-collapsed' if 'data-searchbar-collapsed-value="true"' in element else 'searchbar'
url = ('/en/' if path.startswith('/en/') else '/') + variant
start = m.group(0)
return start[:-1] + SEARCHBAR_ISLAND_ATTRIBUTES.format(url=url) + '>' + element[len(start):]
def snowreport_island(self, text, path, m, element):
"""The placeholder loading the snow report box from TYPO3 13, for the content element of
the anchor right before it; without one the box stays as exported."""
anchor = CONTENT_ANCHOR_RE.search(text, max(0, m.start() - 32), m.start())
if not anchor:
self.error('%s: snow report without content element anchor' % path)
return element
return SNOWREPORT_ISLAND.format(url=html.escape(path.rstrip('/') + '/snowreport'), uid=anchor.group(1))
def process(self, text): def process(self, text):
"""Replace AJAX table URLs (recursively fetching them) and collect assets.""" """Replace AJAX table URLs (recursively fetching them) and collect assets."""
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
@@ -803,9 +888,6 @@ API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JS
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n' REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
STATIC_REWRITE = r''' STATIC_REWRITE = r'''
# Static export: reject multipart requests (CVE-2026-109618 / -109619)
RewriteCond %{HTTP:Content-Type} multipart [NC]
RewriteRule ^ - [F,L]
# No PHP is ever served, also not via a symlinked fileadmin # No PHP is ever served, also not via a symlinked fileadmin
# (/buchung/ is exempt: it redirects to the booking system further down) # (/buchung/ is exempt: it redirects to the booking system further down)
RewriteCond %{REQUEST_URI} !^/buchung/ RewriteCond %{REQUEST_URI} !^/buchung/
@@ -826,9 +908,6 @@ Options -Indexes -MultiViews
''' '''
HYBRID_REWRITE = r''' HYBRID_REWRITE = r'''
# Reject multipart requests (CVE-2026-109618 / -109619); the search sends URL-encoded forms
RewriteCond %{HTTP:Content-Type} multipart [NC]
RewriteRule ^ - [F,L]
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below # No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
# (/buchung/ is exempt: it redirects to the booking system further down) # (/buchung/ is exempt: it redirects to the booking system further down)
RewriteCond %{REQUEST_URI} !^/buchung/ RewriteCond %{REQUEST_URI} !^/buchung/
@@ -921,7 +1000,9 @@ def main():
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing))) % (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search'])) print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
if args.hybrid: if args.hybrid:
print(' left to TYPO3 13: %d pages' % exp.stats['dynamic']) print(' left to TYPO3 13: %d pages, forms loaded from TYPO3 13: %d, snow reports loaded from TYPO3 13: %d,'
' search bars refreshed from TYPO3 13: %d'
% (exp.stats['dynamic'], exp.stats['form_islands'], exp.stats['snowreports'], exp.stats['searchbars']))
for path in sorted(missing)[:20]: for path in sorted(missing)[:20]:
print(' missing asset: %s' % path) print(' missing asset: %s' % path)
for msg in exp.errors[:20]: for msg in exp.errors[:20]:
+1
View File
@@ -21,3 +21,4 @@ public/typo3temp
public/uploads public/uploads
var var
http-client.private.env.json http-client.private.env.json
.ddev/static-export/__pycache__/export.cpython-313.pyc