feat: hybrid export loads EXT:form forms from TYPO3 13, multipart
allowed
This commit is contained in:
1 parent
f4f0857870
commit
85b1bd1578
3 files changed
+99
-17
No files matched your search
Binary file not shown.
@@ -8,7 +8,7 @@ plain document roots that run without PHP.
|
|||||||
|
|
||||||
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
|
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
|
||||||
and the search are left out (TYPO3 13 serves them through index.php), the search
|
and the search are left out (TYPO3 13 serves them through index.php), the search
|
||||||
UI stays (except the search bar), and the API is answered live.
|
UI stays (the search bar is refreshed from TYPO3 13), and the API is answered live.
|
||||||
|
|
||||||
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
|
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
|
||||||
[--hybrid --typo3-current PATH [--live-ajax]]
|
[--hybrid --typo3-current PATH [--live-ajax]]
|
||||||
@@ -45,6 +45,46 @@ AJAX_PATH = '/ajax/'
|
|||||||
# cannot work statically and are replaced by a contact notice
|
# cannot work statically and are replaced by a contact notice
|
||||||
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
|
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
|
||||||
|
|
||||||
|
# Hybrid: EXT:form plugins (ep_theme Templates/Form/Render.html: <div class="py-8"><form ...>) are
|
||||||
|
# loaded from TYPO3 13 instead, from <page>/form (page type 1704, renders the page's form only).
|
||||||
|
# htmx swaps the placeholder on load and the form posts back there. ?ref= (the page a visitor came
|
||||||
|
# from) is passed along for the enquiry e-mails. Until the form arrives the placeholder shows the
|
||||||
|
# spinner of the form islands (ep_theme Partials/Form/Spinner.html, here in the text colour: the
|
||||||
|
# theme colours come with the island). The contact notice is shown without JavaScript and, taken
|
||||||
|
# from the noscript element, when TYPO3 13 does not answer.
|
||||||
|
FORM_PLUGIN_RE = re.compile(r'<div class="py-8">(?=\s*<form\b[^>]*\baction="[^"]*tx_form_formframework)', re.I)
|
||||||
|
FORM_ISLAND_FALLBACK = "this.removeAttribute('style'); this.innerHTML = this.querySelector('noscript').textContent"
|
||||||
|
FORM_SPINNER = (
|
||||||
|
'<svg style="width:4rem;height:4rem;opacity:.5" viewBox="0 0 44 44" xmlns="http://www.w3.org/2000/svg" stroke="currentColor">'
|
||||||
|
'<g fill="none" fill-rule="evenodd" stroke-width="2">'
|
||||||
|
+ ''.join('<circle cx="22" cy="22" r="1">'
|
||||||
|
'<animate attributeName="r" begin="%s" dur="1.8s" values="1; 20" calcMode="spline" keyTimes="0; 1" keySplines="0.165, 0.84, 0.44, 1" repeatCount="indefinite" />'
|
||||||
|
'<animate attributeName="stroke-opacity" begin="%s" dur="1.8s" values="1; 0" calcMode="spline" keyTimes="0; 1" keySplines="0.3, 0.61, 0.355, 1" repeatCount="indefinite" />'
|
||||||
|
'</circle>' % (begin, begin) for begin in ('0s', '-0.9s'))
|
||||||
|
+ '</g></svg>')
|
||||||
|
FORM_ISLAND = ('<div class="form-island" hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
|
||||||
|
' hx-vals=\'js:{{ref: new URLSearchParams(location.search).get("ref") || ""}}\''
|
||||||
|
' hx-on:htmx:response-error="' + FORM_ISLAND_FALLBACK + '"'
|
||||||
|
' hx-on:htmx:send-error="' + FORM_ISLAND_FALLBACK + '"'
|
||||||
|
' style="min-height:12rem;display:flex;align-items:center;justify-content:center">'
|
||||||
|
+ FORM_SPINNER + '<noscript>{notice}</noscript></div>')
|
||||||
|
|
||||||
|
# Hybrid: snow report lists (ep_theme Templates/Snowreport/List.html: a column of rows starting
|
||||||
|
# with flag and region, in the spacing wrapper of ep:container if the element has spacing) are
|
||||||
|
# loaded from TYPO3 13, from <page>/snowreport (page type 1705, renders the page's snow reports
|
||||||
|
# only, each in id="snowreport-<content uid>" around the spacing wrapper). The export keeps an
|
||||||
|
# empty placeholder only: no outdated snow depths, and nothing without JavaScript. When TYPO3 13
|
||||||
|
# does not answer or no longer has that list, the placeholder is removed.
|
||||||
|
SNOWREPORT_BOX_RE = re.compile(
|
||||||
|
r'(?<=</a>)\s*<div class="[^"]*">'
|
||||||
|
r'(?=(?:\s*<div class="flex flex-col space-y-2">)?\s*<div class="grid md:grid-cols-3 md:gap-y-0">'
|
||||||
|
r'\s*<div class="md:col-span-1 bg-ep-primary-dark p-2 flex items-start space-x-2">)', re.I)
|
||||||
|
CONTENT_ANCHOR_RE = re.compile(r'<a id="c(\d+)"></a>$')
|
||||||
|
SNOWREPORT_ISLAND = ('<div hx-get="{url}" hx-trigger="load" hx-select="#snowreport-{uid}" hx-swap="outerHTML"'
|
||||||
|
' hx-on:htmx:before-swap="event.detail.shouldSwap = event.detail.xhr.responseText'
|
||||||
|
'.includes(\'id="snowreport-{uid}"\'); event.detail.shouldSwap || this.remove()"'
|
||||||
|
' hx-on:htmx:send-error="this.remove()"></div>')
|
||||||
|
|
||||||
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
|
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
|
||||||
# and links to the search page (menu icon, "Alle anzeigen" with preset filters)
|
# and links to the search page (menu icon, "Alle anzeigen" with preset filters)
|
||||||
SEARCH_ELEMENTS = (
|
SEARCH_ELEMENTS = (
|
||||||
@@ -55,12 +95,19 @@ SEARCH_ELEMENTS = (
|
|||||||
)
|
)
|
||||||
SEARCH_PAGE_RE = re.compile(r'/suche/$')
|
SEARCH_PAGE_RE = re.compile(r'/suche/$')
|
||||||
|
|
||||||
# Hybrid: only the search bar goes (TYPO3 13 has no search bar plugin); links, menu icons and
|
# Hybrid: the search UI stays, the search itself is served by TYPO3 13. The search bar
|
||||||
# the mobile filter overlay stay, the search itself is served by TYPO3 13
|
# (ep_theme Templates/Searchbar/Index.html) is replaced on load by the current one from TYPO3 13,
|
||||||
HYBRID_SEARCH_ELEMENTS = (
|
# from <language root>searchbar or searchbar-collapsed (page types 1706/1707, the same on every
|
||||||
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
|
# page). Until then, and when TYPO3 13 does not answer, the exported bar stays: its form is
|
||||||
('button', re.compile(r'<button\b[^>]*\bdata-controller="searchbar-toggle"[^>]*>')),
|
# rewritten to the GET form of TYPO3 13 (fields as SearchController::SEARCHBAR_PARAMETERS),
|
||||||
)
|
# since TYPO3 13 has no processSearch and would reject the signed fields of TYPO3 10.
|
||||||
|
HYBRID_SEARCH_ELEMENTS = ()
|
||||||
|
# The whole start tag: its data-action contains "->"
|
||||||
|
SEARCHBAR_RE = re.compile(r'<div\b(?:[^>"]|"[^"]*")*?\sdata-controller="searchbar"(?:[^>"]|"[^"]*")*>')
|
||||||
|
SEARCHBAR_FORM_RE = re.compile(r'<form data-searchbar-target="form" method="post" name="searchParams" action="([^"]*)">')
|
||||||
|
SEARCHBAR_SIGNED_FIELDS_RE = re.compile(r'<div>(?:\s*<input type="hidden" name="tx_epproducts_searchresult\[__[^>]*>)+\s*</div>')
|
||||||
|
SEARCHBAR_FIELD_RE = re.compile(r'name="tx_epproducts_searchresult\[searchParams\]\[(\w+)\]"')
|
||||||
|
SEARCHBAR_ISLAND_ATTRIBUTES = ' hx-get="{url}" hx-trigger="load" hx-swap="outerHTML"'
|
||||||
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
|
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
|
||||||
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
|
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
|
||||||
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
|
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
|
||||||
@@ -103,7 +150,8 @@ CONTACT_NOTICE = '''<div class="static-contact-notice" style="background:#ebf3fa
|
|||||||
|
|
||||||
|
|
||||||
def replace_elements(text, tag, start_re, replacement):
|
def replace_elements(text, tag, start_re, replacement):
|
||||||
"""Replace each element whose start tag matches, including its balanced content."""
|
"""Replace each element whose start tag matches, including its balanced content. replacement
|
||||||
|
may be a function of the start tag match and the element, returning the new markup."""
|
||||||
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
|
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
|
||||||
out, pos, count = [], 0, 0
|
out, pos, count = [], 0, 0
|
||||||
for m in start_re.finditer(text):
|
for m in start_re.finditer(text):
|
||||||
@@ -117,7 +165,7 @@ def replace_elements(text, tag, start_re, replacement):
|
|||||||
break
|
break
|
||||||
if end is None:
|
if end is None:
|
||||||
continue # unbalanced markup, leave it alone
|
continue # unbalanced markup, leave it alone
|
||||||
out += [text[pos:m.start()], replacement]
|
out += [text[pos:m.start()], replacement(m, text[m.start():end]) if callable(replacement) else replacement]
|
||||||
pos, count = end, count + 1
|
pos, count = end, count + 1
|
||||||
out.append(text[pos:])
|
out.append(text[pos:])
|
||||||
return ''.join(out), count
|
return ''.join(out), count
|
||||||
@@ -200,7 +248,7 @@ class Exporter:
|
|||||||
self.errors = []
|
self.errors = []
|
||||||
self.crawl_redirects = []
|
self.crawl_redirects = []
|
||||||
self.follow_redirects = follow
|
self.follow_redirects = follow
|
||||||
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'search': 0, 'dynamic': 0}
|
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'form_islands': 0, 'snowreports': 0, 'searchbars': 0, 'search': 0, 'dynamic': 0}
|
||||||
|
|
||||||
# --- helpers -----------------------------------------------------------
|
# --- helpers -----------------------------------------------------------
|
||||||
|
|
||||||
@@ -301,6 +349,14 @@ class Exporter:
|
|||||||
phone, tel, email = self.contact
|
phone, tel, email = self.contact
|
||||||
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
|
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
|
||||||
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
|
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
|
||||||
|
islands = snowreports = searchbars = 0
|
||||||
|
if self.hybrid and path:
|
||||||
|
text, searchbars = replace_elements(text, 'div', SEARCHBAR_RE,
|
||||||
|
lambda m, element: self.searchbar_island(path, m, element))
|
||||||
|
island = FORM_ISLAND.format(url=html.escape(path.rstrip('/') + '/form'), notice=notice)
|
||||||
|
text, islands = replace_elements(text, 'div', FORM_PLUGIN_RE, island)
|
||||||
|
text, snowreports = replace_elements(text, 'div', SNOWREPORT_BOX_RE,
|
||||||
|
lambda m, element: self.snowreport_island(text, path, m, element))
|
||||||
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
|
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
|
||||||
removed = 0
|
removed = 0
|
||||||
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
|
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
|
||||||
@@ -308,9 +364,38 @@ class Exporter:
|
|||||||
removed += n
|
removed += n
|
||||||
with self.lock:
|
with self.lock:
|
||||||
self.stats['forms'] += forms
|
self.stats['forms'] += forms
|
||||||
|
self.stats['form_islands'] += islands
|
||||||
|
self.stats['snowreports'] += snowreports
|
||||||
|
self.stats['searchbars'] += searchbars
|
||||||
self.stats['search'] += removed
|
self.stats['search'] += removed
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
def searchbar_island(self, path, m, element):
|
||||||
|
"""The exported search bar with the GET form of TYPO3 13, replaced on load by the
|
||||||
|
current bar from TYPO3 13 (open or collapsed, as exported)."""
|
||||||
|
element, forms = SEARCHBAR_FORM_RE.subn(
|
||||||
|
lambda f: '<form action="%s" method="get" data-searchbar-target="form">'
|
||||||
|
% html.escape(html.unescape(f.group(1)).split('?')[0]), element) # host: see process()
|
||||||
|
element, signed = SEARCHBAR_SIGNED_FIELDS_RE.subn('', element)
|
||||||
|
element, fields = SEARCHBAR_FIELD_RE.subn(r'name="\1"', element)
|
||||||
|
if (forms, signed, fields) != (1, 1, 6):
|
||||||
|
self.error('%s: search bar form not as expected (form %d, signed fields %d, fields %d), left out'
|
||||||
|
% (path, forms, signed, fields))
|
||||||
|
return ''
|
||||||
|
variant = 'searchbar-collapsed' if 'data-searchbar-collapsed-value="true"' in element else 'searchbar'
|
||||||
|
url = ('/en/' if path.startswith('/en/') else '/') + variant
|
||||||
|
start = m.group(0)
|
||||||
|
return start[:-1] + SEARCHBAR_ISLAND_ATTRIBUTES.format(url=url) + '>' + element[len(start):]
|
||||||
|
|
||||||
|
def snowreport_island(self, text, path, m, element):
|
||||||
|
"""The placeholder loading the snow report box from TYPO3 13, for the content element of
|
||||||
|
the anchor right before it; without one the box stays as exported."""
|
||||||
|
anchor = CONTENT_ANCHOR_RE.search(text, max(0, m.start() - 32), m.start())
|
||||||
|
if not anchor:
|
||||||
|
self.error('%s: snow report without content element anchor' % path)
|
||||||
|
return element
|
||||||
|
return SNOWREPORT_ISLAND.format(url=html.escape(path.rstrip('/') + '/snowreport'), uid=anchor.group(1))
|
||||||
|
|
||||||
def process(self, text):
|
def process(self, text):
|
||||||
"""Replace AJAX table URLs (recursively fetching them) and collect assets."""
|
"""Replace AJAX table URLs (recursively fetching them) and collect assets."""
|
||||||
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
|
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
|
||||||
@@ -803,9 +888,6 @@ API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JS
|
|||||||
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
|
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
|
||||||
|
|
||||||
STATIC_REWRITE = r'''
|
STATIC_REWRITE = r'''
|
||||||
# Static export: reject multipart requests (CVE-2026-109618 / -109619)
|
|
||||||
RewriteCond %{HTTP:Content-Type} multipart [NC]
|
|
||||||
RewriteRule ^ - [F,L]
|
|
||||||
# No PHP is ever served, also not via a symlinked fileadmin
|
# No PHP is ever served, also not via a symlinked fileadmin
|
||||||
# (/buchung/ is exempt: it redirects to the booking system further down)
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
||||||
RewriteCond %{REQUEST_URI} !^/buchung/
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
||||||
@@ -826,9 +908,6 @@ Options -Indexes -MultiViews
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
HYBRID_REWRITE = r'''
|
HYBRID_REWRITE = r'''
|
||||||
# Reject multipart requests (CVE-2026-109618 / -109619); the search sends URL-encoded forms
|
|
||||||
RewriteCond %{HTTP:Content-Type} multipart [NC]
|
|
||||||
RewriteRule ^ - [F,L]
|
|
||||||
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
|
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
|
||||||
# (/buchung/ is exempt: it redirects to the booking system further down)
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
||||||
RewriteCond %{REQUEST_URI} !^/buchung/
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
||||||
@@ -921,7 +1000,9 @@ def main():
|
|||||||
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
|
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
|
||||||
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
|
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
|
||||||
if args.hybrid:
|
if args.hybrid:
|
||||||
print(' left to TYPO3 13: %d pages' % exp.stats['dynamic'])
|
print(' left to TYPO3 13: %d pages, forms loaded from TYPO3 13: %d, snow reports loaded from TYPO3 13: %d,'
|
||||||
|
' search bars refreshed from TYPO3 13: %d'
|
||||||
|
% (exp.stats['dynamic'], exp.stats['form_islands'], exp.stats['snowreports'], exp.stats['searchbars']))
|
||||||
for path in sorted(missing)[:20]:
|
for path in sorted(missing)[:20]:
|
||||||
print(' missing asset: %s' % path)
|
print(' missing asset: %s' % path)
|
||||||
for msg in exp.errors[:20]:
|
for msg in exp.errors[:20]:
|
||||||
|
|||||||
@@ -21,3 +21,4 @@ public/typo3temp
|
|||||||
public/uploads
|
public/uploads
|
||||||
var
|
var
|
||||||
http-client.private.env.json
|
http-client.private.env.json
|
||||||
|
.ddev/static-export/__pycache__/export.cpython-313.pyc
|
||||||
Reference in new issue
Block a user