chore: add exporter scripts for static clones
This commit is contained in:
1 parent
2771be2d67
commit
f4f0857870
7 files changed
+1114
No files matched your search
Executable
+7
@@ -0,0 +1,7 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
## Description: Export the TYPO3 sites as static document roots (no PHP)
|
||||||
|
## Usage: static-export [--site <identifier>] [--url <ddev url>] [--no-follow] [--out <dir>] [--workers <n>] [--hybrid --typo3-current <path> [--live-ajax]]
|
||||||
|
## Example: "ddev static-export --site ep-reisen"
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
exec python3 -I "${DDEV_APPROOT}/.ddev/static-export/export.py" "$@"
|
||||||
Binary file not shown.
@@ -0,0 +1,36 @@
|
|||||||
|
"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files."""
|
||||||
|
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections
|
||||||
|
root, prod = sys.argv[1], sys.argv[2].rstrip('/')
|
||||||
|
host = urllib.parse.urlsplit(prod).netloc
|
||||||
|
class NoRedir(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *a): return None
|
||||||
|
op = urllib.request.build_opener(NoRedir)
|
||||||
|
def get(url, body=False):
|
||||||
|
try:
|
||||||
|
with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r:
|
||||||
|
return r.status, (r.read().decode('utf-8', 'replace') if body else '')
|
||||||
|
except urllib.error.HTTPError as e: return e.code, ''
|
||||||
|
except Exception as e: return str(e)[:60], ''
|
||||||
|
pages = []
|
||||||
|
for d, _, files in os.walk(root):
|
||||||
|
if '/_ajax' in d: continue
|
||||||
|
for f in files:
|
||||||
|
if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else ''))
|
||||||
|
pages = sorted(set(p.replace('//', '/') for p in pages))
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(12) as ex:
|
||||||
|
res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages)))
|
||||||
|
refs = set()
|
||||||
|
for p, (st, body) in res.items():
|
||||||
|
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body):
|
||||||
|
for c in html.unescape(v).split(','):
|
||||||
|
c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c)
|
||||||
|
if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'):
|
||||||
|
base = 'https://' + (u.netloc or host)
|
||||||
|
refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else ''))
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
||||||
|
rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs))))
|
||||||
|
print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values()))))
|
||||||
|
by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items())
|
||||||
|
print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)]
|
||||||
|
bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))]
|
||||||
|
for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u))
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""Request every exported page and every local URL referenced in pages/fragments via a local Apache."""
|
||||||
|
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures
|
||||||
|
root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099'
|
||||||
|
class NoRedir(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *a): return None
|
||||||
|
op = urllib.request.build_opener(NoRedir)
|
||||||
|
def get(path):
|
||||||
|
try:
|
||||||
|
with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status
|
||||||
|
except urllib.error.HTTPError as e: return e.code
|
||||||
|
except Exception as e: return str(e)
|
||||||
|
pages, refs = [], set()
|
||||||
|
for d, _, files in os.walk(root):
|
||||||
|
for f in files:
|
||||||
|
if not f.endswith('.html'): continue
|
||||||
|
rel = '/' + os.path.relpath(os.path.join(d, f), root)
|
||||||
|
if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel)
|
||||||
|
t = open(os.path.join(d, f), encoding='utf-8').read()
|
||||||
|
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t):
|
||||||
|
for c in html.unescape(v).split(','):
|
||||||
|
c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c)
|
||||||
|
if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'):
|
||||||
|
refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else ''))
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
||||||
|
pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs))))
|
||||||
|
bad_p = {k: v for k, v in pr.items() if v != 200}
|
||||||
|
bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)}
|
||||||
|
print('pages: %d, not 200: %d' % (len(pr), len(bad_p)))
|
||||||
|
print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r)))
|
||||||
|
for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""Compare /api/hotel + /api/product of a static export (Apache on 127.0.0.1:8099) with live local TYPO3.
|
||||||
|
|
||||||
|
Usage: compare_api.py <export dir>. Case list: every hotel/product code, sampled pairs, random
|
||||||
|
unlinked pairs, empty/missing params, missing/wrong/query-string key, direct snapshot access.
|
||||||
|
"""
|
||||||
|
import sys, os, random, importlib.util, urllib.request, urllib.error, concurrent.futures
|
||||||
|
spec = importlib.util.spec_from_file_location('export', '/var/www/html/.ddev/static-export/export.py'); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||||
|
sites = m.load_sites(); exp = m.Exporter('ep-reisen', sites['ep-reisen'], sites, '/nonexistent', follow=False, workers=1); key = m.cms_api_key()
|
||||||
|
class N(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *a): return None
|
||||||
|
op = urllib.request.build_opener(N)
|
||||||
|
def call(url, headers):
|
||||||
|
try:
|
||||||
|
with op.open(urllib.request.Request(url, headers=headers), timeout=60) as r: return r.status, r.read().decode(), r.headers.get('Content-Type')
|
||||||
|
except urllib.error.HTTPError as e: return e.code, e.read().decode(), e.headers.get('Content-Type')
|
||||||
|
typo3 = lambda q, h: call('http://127.0.0.1/' + q, dict(h, **{'Host': 'ep-reisen.ddev.site', 'X-Forwarded-Proto': 'https'}))
|
||||||
|
static = lambda q, h: call('http://127.0.0.1:8099/' + q, dict(h, **{'Host': 'www.ep-reisen.de'}))
|
||||||
|
root = sys.argv[1]; K = {'X-Api-Key': key}
|
||||||
|
hotels = sorted(f[:-5] for f in os.listdir(root + '/_api/hotel')); products = sorted(f[:-5] for f in os.listdir(root + '/_api/product') if f.endswith('.json'))
|
||||||
|
pairs = sorted((p, f[:-5]) for p in products if os.path.isdir(root + '/_api/product/' + p) for f in os.listdir(root + '/_api/product/' + p) if not f.startswith('_'))
|
||||||
|
random.seed(1)
|
||||||
|
cases = [('api/hotel?hotel=' + c, K) for c in hotels] + [('api/product?product=' + c, K) for c in products]
|
||||||
|
cases += [('api/product?product=%s&hotel=%s' % pc, K) for pc in random.sample(pairs, 150)]
|
||||||
|
cases += [('api/product?hotel=%s&product=%s' % (h, p), K) for p, h in random.sample(pairs, 10)]
|
||||||
|
cases += [('api/product?product=%s&hotel=%s' % (p, random.choice(hotels)), K) for p in random.sample(products, 60)]
|
||||||
|
cases += [(q, K) for q in ['api/hotel?hotel=_empty', 'api/product?product=_empty', 'api/hotel?hotel=%40empty', 'api/product?product=3VIIN&hotel=@empty', 'api/product?product=3VIIN&hotel=PMVSW%26A', 'api/hotel?hotel=a%20b', 'api/product?product=%40empty&hotel=', 'api/hotel', 'api/product', 'api/hotel?hotel=', 'api/product?product=', 'api/product?product=&hotel=X',
|
||||||
|
'api/hotel?hotel=NOPE', 'api/product?product=NOPE', 'api/product?product=NOPE&hotel=' + hotels[0],
|
||||||
|
'api/product?product=%s&hotel=' % products[0], 'api/product?product=%s&hotel=NOPE' % products[0]]]
|
||||||
|
cases += [('api/hotel?hotel=' + hotels[0], {}), ('api/hotel?hotel=' + hotels[0], {'X-Api-Key': 'wrong'}),
|
||||||
|
('api/hotel?hotel=%s&key=%s' % (hotels[0], key), {}), ('api/product?key=%s&product=%s' % (key, products[0]), {}),
|
||||||
|
('api/product?product=%s&key=wrong' % products[0], {})]
|
||||||
|
def compare(c):
|
||||||
|
q, h = c; a, b = typo3(q, h), static(q, h)
|
||||||
|
return q, (a[0], exp.to_prod(a[1]), a[2]), b
|
||||||
|
bad = []
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(8) as ex:
|
||||||
|
for q, a, b in ex.map(compare, cases):
|
||||||
|
if a != b: bad.append((q, a, b))
|
||||||
|
print('cases: %d, identical: %d, different: %d' % (len(cases), len(cases) - len(bad), len(bad)))
|
||||||
|
for q, a, b in bad[:6]: print(' DIFF', q, '| typo3', a[0], a[2], '| static', b[0], b[2], repr(b[1][:90]))
|
||||||
|
for q in ['_api/hotel/%s.json' % hotels[0], '_api/pair.shtml', '_api/product-part/%s.part' % products[0]]: print(' direct', q, static(q, {})[0])
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* Import the same buspro data twice (XML files, MyE&P API) into the local DB and keep each
|
||||||
|
* result as cmp_<source>_<table>. The live tables are backed up first and restored at the end.
|
||||||
|
*/
|
||||||
|
putenv('TYPO3_PATH_ROOT=/var/www/html/public');
|
||||||
|
putenv('TYPO3_PATH_APP=/var/www/html');
|
||||||
|
$classLoader = require '/var/www/html/vendor/autoload.php';
|
||||||
|
\TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(0, \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::REQUESTTYPE_CLI);
|
||||||
|
\TYPO3\CMS\Core\Core\Bootstrap::init($classLoader);
|
||||||
|
|
||||||
|
use EP\EpProducts\Service\ApiImportSource;
|
||||||
|
use EP\EpProducts\Service\DateImportService;
|
||||||
|
use EP\EpProducts\Service\XmlProductReader;
|
||||||
|
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||||||
|
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||||
|
|
||||||
|
$db = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
|
||||||
|
$tables = ['date' => 'tx_epproducts_domain_model_date', 'room' => 'tx_epproducts_domain_model_room', 'product' => 'tx_epproducts_domain_model_product'];
|
||||||
|
|
||||||
|
$snapshot = function (string $prefix) use ($db, $tables) {
|
||||||
|
foreach ($tables as $short => $table) {
|
||||||
|
$db->executeStatement("DROP TABLE IF EXISTS cmp_{$prefix}_{$short}");
|
||||||
|
$db->executeStatement("CREATE TABLE cmp_{$prefix}_{$short} AS SELECT * FROM {$table}");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
$restore = function () use ($db, $tables) {
|
||||||
|
foreach ($tables as $short => $table) {
|
||||||
|
$db->executeStatement("DELETE FROM {$table}");
|
||||||
|
$db->executeStatement("INSERT INTO {$table} SELECT * FROM cmp_bak_{$short}");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
$snapshot('bak');
|
||||||
|
try {
|
||||||
|
$reader = new XmlProductReader(GeneralUtility::getFileAbsFileName('fileadmin/xmlexport'));
|
||||||
|
$rows = iterator_to_array($reader, false);
|
||||||
|
$count = GeneralUtility::makeInstance(DateImportService::class)->import($rows);
|
||||||
|
printf("xml: %d rows, %d dates imported, source generated at %s\n", count($rows), $count,
|
||||||
|
$reader->getSourceGeneratedAt() ? $reader->getSourceGeneratedAt()->format('c') : '?');
|
||||||
|
$snapshot('xml');
|
||||||
|
$restore();
|
||||||
|
|
||||||
|
// Bypass the "already imported" baseline: walk the export pages directly
|
||||||
|
$api = GeneralUtility::makeInstance(ApiImportSource::class);
|
||||||
|
$walk = new \ReflectionMethod($api, 'walkPages');
|
||||||
|
$walk->setAccessible(true);
|
||||||
|
$export = $walk->invoke($api);
|
||||||
|
if (!is_array($export)) {
|
||||||
|
throw new \RuntimeException('MyE&P export walk returned ' . var_export($export, true));
|
||||||
|
}
|
||||||
|
[$apiRows, $generatedAt] = $export;
|
||||||
|
$count = GeneralUtility::makeInstance(DateImportService::class)->import($apiRows);
|
||||||
|
printf("api: %d rows, %d dates imported, source generated at %s\n", count($apiRows), $count,
|
||||||
|
$generatedAt ? $generatedAt->format('c') : '?');
|
||||||
|
$snapshot('api');
|
||||||
|
|
||||||
|
// Raw rows of both sources, for a field-by-field comparison independent of the import
|
||||||
|
file_put_contents('/var/www/html/var/rows_xml.json', json_encode($rows));
|
||||||
|
file_put_contents('/var/www/html/var/rows_api.json', json_encode($apiRows));
|
||||||
|
} finally {
|
||||||
|
$restore();
|
||||||
|
echo "live tables restored\n";
|
||||||
|
}
|
||||||
@@ -0,0 +1,936 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Static export of the TYPO3 sites running in DDEV.
|
||||||
|
|
||||||
|
Crawls each site (sitemap + internal links), pre-fetches the AJAX dates and
|
||||||
|
price tables of product pages, rewrites *.ddev.site hosts to the production
|
||||||
|
domains and copies every referenced static asset. The result is a set of
|
||||||
|
plain document roots that run without PHP.
|
||||||
|
|
||||||
|
With --hybrid the document roots sit in front of TYPO3 13 instead: product pages
|
||||||
|
and the search are left out (TYPO3 13 serves them through index.php), the search
|
||||||
|
UI stays (except the search bar), and the API is answered live.
|
||||||
|
|
||||||
|
Usage: export.py [--site ep-reisen] [--url URL ...] [--no-follow] [--out DIR]
|
||||||
|
[--hybrid --typo3-current PATH [--live-ajax]]
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
import concurrent.futures
|
||||||
|
import hashlib
|
||||||
|
import html
|
||||||
|
import http.client
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
APP_ROOT = os.environ.get('DDEV_APPROOT', '/var/www/html')
|
||||||
|
PUBLIC = os.path.join(APP_ROOT, 'public')
|
||||||
|
SITES_DIR = os.path.join(APP_ROOT, 'config', 'sites')
|
||||||
|
|
||||||
|
# Hosts that pull from the production origin; their paths must exist in the export
|
||||||
|
CDN_HOSTS = {'cdn.ep-reisen.de'}
|
||||||
|
|
||||||
|
# Any attribute pointing at TYPO3's AJAX page type (product tables, HTMX teaser
|
||||||
|
# popups, ...) is pre-fetched; JS may append "&pa=..." to these URLs
|
||||||
|
AJAX_PATH = '/ajax/'
|
||||||
|
|
||||||
|
# Forms posting to these TYPO3 plugins (contact, enquiry, application, booking, newsletter)
|
||||||
|
# cannot work statically and are replaced by a contact notice
|
||||||
|
CONTACT_FORM_RE = re.compile(r'<form\b[^>]*\bmethod="post"[^>]*\baction="[^"]*(?:tx_form_formframework|newsletter_subscription)[^>]*>', re.I)
|
||||||
|
|
||||||
|
# Search UI, removed entirely: search bar, mobile search overlay, the buttons opening them,
|
||||||
|
# and links to the search page (menu icon, "Alle anzeigen" with preset filters)
|
||||||
|
SEARCH_ELEMENTS = (
|
||||||
|
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
|
||||||
|
('div', re.compile(r'<div\b[^>]*\bdata-controller="mobile-search"[^>]*>')),
|
||||||
|
('button', re.compile(r'<button\b[^>]*\bdata-overlay-toggle-id-value="(?:mobile-)?search"[^>]*>')),
|
||||||
|
('a', re.compile(r'<a\b[^>]*\bhref="[^"]*/suche/(?:\?[^"]*)?"[^>]*>')),
|
||||||
|
)
|
||||||
|
SEARCH_PAGE_RE = re.compile(r'/suche/$')
|
||||||
|
|
||||||
|
# Hybrid: only the search bar goes (TYPO3 13 has no search bar plugin); links, menu icons and
|
||||||
|
# the mobile filter overlay stay, the search itself is served by TYPO3 13
|
||||||
|
HYBRID_SEARCH_ELEMENTS = (
|
||||||
|
('div', re.compile(r'<div\b[^>]*\bdata-controller="searchbar"[^>]*>')),
|
||||||
|
('button', re.compile(r'<button\b[^>]*\bdata-controller="searchbar-toggle"[^>]*>')),
|
||||||
|
)
|
||||||
|
# Hybrid --live-ajax: controller/action pairs TYPO3 13 serves (compared with TYPO3 10 on
|
||||||
|
# 2026-10-21); every other AJAX URL is still pre-fetched. Not AjaxTeaserPopup: its targetUrl
|
||||||
|
# argument is a full URL, and to_prod() rewriting the dev host inside it invalidates the cHash
|
||||||
|
# (404 on TYPO3 13 for popups pointing at another site)
|
||||||
|
LIVE_AJAX = {
|
||||||
|
('AjaxTable', 'datestable'), ('AjaxTable', 'pricetable'), ('AjaxTable', 'pricetableHtml'),
|
||||||
|
('AjaxTable', 'eventPricetable'),
|
||||||
|
('AjaxCalendar', 'priceTable'), ('AjaxCalendar', 'priceCalendar'),
|
||||||
|
('AjaxSearch', 'searchresult'), ('AjaxFilterPanel', 'index'), ('AjaxWatchlist', 'list'),
|
||||||
|
}
|
||||||
|
LIVE_AJAX_PARAM_RE = re.compile(r'tx_epproducts_ajax(?:%5B|\[)(controller|action)(?:%5D|\])=(\w+)')
|
||||||
|
|
||||||
|
# Contact shown instead of forms: display phone, tel: number, e-mail (as in each site's footer)
|
||||||
|
CONTACTS = {
|
||||||
|
'ep-reisen': ('0221 272276-0', '+492212722760', '[email protected]'),
|
||||||
|
'ski-boarderweek': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
||||||
|
'snowzone': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
||||||
|
'unichamp': ('0221 272276-47', '+4922127227647', '[email protected]'),
|
||||||
|
'ser': ('0221 272276-41', '+4922127227641', '[email protected]'),
|
||||||
|
'alvac': ('0221 272276-30', '+4922127227630', '[email protected]'),
|
||||||
|
'cllt': ('0221 272276-30', '+4922127227630', '[email protected]'),
|
||||||
|
}
|
||||||
|
DEFAULT_CONTACT = CONTACTS['ep-reisen']
|
||||||
|
|
||||||
|
CONTACT_TEXT = {
|
||||||
|
'de': ('Dieses Formular ist vorübergehend nicht verfügbar.',
|
||||||
|
'Du hast Fragen oder möchtest buchen? Unser Kundenservice hilft dir gerne weiter:'),
|
||||||
|
'en': ('This form is temporarily unavailable.',
|
||||||
|
'Questions or want to book? Our customer service is happy to help:'),
|
||||||
|
}
|
||||||
|
|
||||||
|
CONTACT_NOTICE = '''<div class="static-contact-notice" style="background:#ebf3fa;border-left:4px solid #3d8ccb;padding:24px;margin:16px 0">
|
||||||
|
<p style="margin:0 0 8px;font-weight:700">{lead}</p>
|
||||||
|
<p style="margin:0 0 16px">{text}</p>
|
||||||
|
<p style="margin:0;display:flex;flex-wrap:wrap;gap:8px">
|
||||||
|
<a href="tel:{tel}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{phone}</a>
|
||||||
|
<a href="mailto:{email}" style="display:inline-block;padding:12px 20px;background:#f9c700;color:#18527b;font-weight:700;text-decoration:none">{email}</a>
|
||||||
|
</p>
|
||||||
|
</div>'''
|
||||||
|
|
||||||
|
|
||||||
|
def replace_elements(text, tag, start_re, replacement):
|
||||||
|
"""Replace each element whose start tag matches, including its balanced content."""
|
||||||
|
token_re = re.compile(r'<(/?)%s\b[^>]*>' % tag, re.I)
|
||||||
|
out, pos, count = [], 0, 0
|
||||||
|
for m in start_re.finditer(text):
|
||||||
|
if m.start() < pos:
|
||||||
|
continue # nested inside an element already replaced
|
||||||
|
depth, end = 1, None
|
||||||
|
for t in token_re.finditer(text, m.end()):
|
||||||
|
depth += -1 if t.group(1) else 1
|
||||||
|
if depth == 0:
|
||||||
|
end = t.end()
|
||||||
|
break
|
||||||
|
if end is None:
|
||||||
|
continue # unbalanced markup, leave it alone
|
||||||
|
out += [text[pos:m.start()], replacement]
|
||||||
|
pos, count = end, count + 1
|
||||||
|
out.append(text[pos:])
|
||||||
|
return ''.join(out), count
|
||||||
|
|
||||||
|
URL_ATTR_RE = re.compile(r'(?:href|src|srcset|data-src|data-srcset|content|poster|action)="([^"]*)"')
|
||||||
|
CSS_URL_RE = re.compile(r'url\(\s*[\'"]?([^\'")]+)[\'"]?\s*\)')
|
||||||
|
ASSET_EXT_RE = re.compile(r'\.(css|js|mjs|json|map|svg|png|jpe?g|gif|webp|avif|ico|woff2?|ttf|eot|otf|pdf|mp4|webm|txt|xml|webmanifest)$', re.I)
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
"""Report redirects instead of following them, so they become rewrite rules."""
|
||||||
|
|
||||||
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
OPENER = urllib.request.build_opener(NoRedirect)
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_rule(path, location, status):
|
||||||
|
pattern = re.escape(urllib.parse.unquote(path).strip('/')).replace('\\ ', '\\x20')
|
||||||
|
target = re.sub(r'([$%])', r'\\\1', location)
|
||||||
|
return '\tRewriteRule "^%s/?$" "%s" [R=%d,L,NE]' % (pattern, target, status)
|
||||||
|
|
||||||
|
|
||||||
|
def load_sites(only=None):
|
||||||
|
"""Read base and Development base variant from each site config."""
|
||||||
|
sites = {}
|
||||||
|
for name in sorted(os.listdir(SITES_DIR)):
|
||||||
|
path = os.path.join(SITES_DIR, name, 'config.yaml')
|
||||||
|
if not os.path.isfile(path) or (only and name not in only):
|
||||||
|
continue
|
||||||
|
with open(path, encoding='utf-8') as fh:
|
||||||
|
text = fh.read()
|
||||||
|
prod = re.search(r"^base:\s*'?([^'\s]+)", text, re.M)
|
||||||
|
dev = re.search(r"base:\s*'?(https://[^'\s]+\.ddev\.site)'?\s*\n\s*condition:\s*'applicationContext == \"Development\"'", text)
|
||||||
|
if prod and dev and prod.group(1).startswith('http'):
|
||||||
|
root = re.search(r'^rootPageId:\s*(\d+)', text, re.M)
|
||||||
|
sites[name] = {'prod': prod.group(1).rstrip('/'), 'dev': dev.group(1).rstrip('/'),
|
||||||
|
'root': int(root.group(1)) if root else 0, 'languages': site_languages(text)}
|
||||||
|
return sites
|
||||||
|
|
||||||
|
|
||||||
|
def site_languages(text):
|
||||||
|
"""languageId => base path prefix ('' for '/', '/en' for '/en/') of a site config."""
|
||||||
|
section = re.search(r'^languages:\n((?:[ \t-].*\n|\n)*)', text, re.M)
|
||||||
|
languages = {}
|
||||||
|
for item in re.split(r'^ -[ \t]*\n?', section.group(1) if section else '', flags=re.M):
|
||||||
|
language_id = re.search(r"^\s*languageId:\s*'?(\d+)", item, re.M)
|
||||||
|
base = re.search(r"^\s{4}base:\s*'?([^'\s]+)", item, re.M)
|
||||||
|
if language_id and base and base.group(1).startswith('/'):
|
||||||
|
languages[int(language_id.group(1))] = base.group(1).rstrip('/')
|
||||||
|
return languages
|
||||||
|
|
||||||
|
|
||||||
|
class Exporter:
|
||||||
|
def __init__(self, name, site, all_sites, out_dir, follow, workers, hybrid=False, live_ajax=False):
|
||||||
|
self.site = site
|
||||||
|
self.hybrid = hybrid
|
||||||
|
self.live_ajax = live_ajax
|
||||||
|
self.dynamic = dynamic_paths(site) if hybrid else set()
|
||||||
|
self.route_prefixes = set(route_pages(site).values()) if hybrid else set()
|
||||||
|
self.contact = CONTACTS.get(name, DEFAULT_CONTACT)
|
||||||
|
self.dev = site['dev']
|
||||||
|
self.dev_host = urllib.parse.urlsplit(self.dev).netloc
|
||||||
|
# Editors often paste absolute production URLs; those are internal links too
|
||||||
|
self.prod_host = urllib.parse.urlsplit(site['prod']).netloc
|
||||||
|
self.ajax_results = {}
|
||||||
|
self.ajax_locks = {}
|
||||||
|
self.out = out_dir
|
||||||
|
self.follow = follow
|
||||||
|
self.workers = workers
|
||||||
|
# dev → prod for every site, so cross-site links come out right too
|
||||||
|
self.host_map = [(s['dev'], s['prod']) for s in all_sites.values()]
|
||||||
|
# Some pages get their AJAX URLs below a parent slug (/reisen-fuer-gruppen/ajax/html?...)
|
||||||
|
self.ajax_attr_re = re.compile(r'([a-zA-Z][\w:.-]*)="((?:%s)?(?:/[\w-]+)*%s[^"]*)"' % (re.escape(self.dev), re.escape(AJAX_PATH)))
|
||||||
|
self.seen = set()
|
||||||
|
self.assets = set()
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.errors = []
|
||||||
|
self.crawl_redirects = []
|
||||||
|
self.follow_redirects = follow
|
||||||
|
self.stats = {'pages': 0, 'ajax': 0, 'redirects': 0, 'redirects_skipped': 0, 'forms': 0, 'search': 0, 'dynamic': 0}
|
||||||
|
|
||||||
|
# --- helpers -----------------------------------------------------------
|
||||||
|
|
||||||
|
def fetch(self, url, extra_headers=None, log_errors=True):
|
||||||
|
"""Return status, body and (for redirects) the Location header."""
|
||||||
|
headers = {'User-Agent': 'ep-static-export'}
|
||||||
|
headers.update(extra_headers or {})
|
||||||
|
parts = urllib.parse.urlsplit(url)
|
||||||
|
if parts.scheme == 'https' and parts.hostname.endswith('.ddev.site'):
|
||||||
|
# Talk to the container's own web server, bypassing ddev-router (works for hostnames
|
||||||
|
# added without "ddev restart"); TYPO3 still sees HTTPS via the forwarded header
|
||||||
|
url = urllib.parse.urlunsplit(('http', '127.0.0.1') + tuple(parts[2:]))
|
||||||
|
headers.update({'Host': parts.hostname, 'X-Forwarded-Proto': 'https'})
|
||||||
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
try:
|
||||||
|
with OPENER.open(req, timeout=60) as res:
|
||||||
|
return res.status, res.read().decode('utf-8', 'replace'), None
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
return e.code, e.read().decode('utf-8', 'replace'), e.headers.get('Location')
|
||||||
|
except Exception as e: # noqa: BLE001 - report and continue
|
||||||
|
if log_errors:
|
||||||
|
self.error('%s: %s' % (url, e))
|
||||||
|
return 0, '', None
|
||||||
|
|
||||||
|
def error(self, msg):
|
||||||
|
with self.lock:
|
||||||
|
self.errors.append(msg)
|
||||||
|
|
||||||
|
def write(self, rel_path, content):
|
||||||
|
path = os.path.join(self.out, rel_path.lstrip('/'))
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, 'w', encoding='utf-8') as fh:
|
||||||
|
fh.write(content)
|
||||||
|
|
||||||
|
def to_prod(self, text):
|
||||||
|
for dev, prod in self.host_map:
|
||||||
|
text = text.replace(dev, prod)
|
||||||
|
text = text.replace(dev.replace('/', '\\/'), prod.replace('/', '\\/'))
|
||||||
|
text = text.replace(urllib.parse.quote(dev, safe=''), urllib.parse.quote(prod, safe=''))
|
||||||
|
return text
|
||||||
|
|
||||||
|
def local_path(self, url):
|
||||||
|
"""Path on this site for a URL, or None if it points elsewhere."""
|
||||||
|
parts = urllib.parse.urlsplit(html.unescape(url))
|
||||||
|
if parts.scheme and parts.scheme not in ('http', 'https'):
|
||||||
|
return None
|
||||||
|
if parts.netloc and parts.netloc not in (self.dev_host, self.prod_host) and parts.netloc not in CDN_HOSTS:
|
||||||
|
return None
|
||||||
|
if not parts.path.startswith('/') or parts.path.startswith('//'):
|
||||||
|
return None
|
||||||
|
return parts
|
||||||
|
|
||||||
|
def collect(self, text):
|
||||||
|
"""Collect asset paths and (if following) internal page links."""
|
||||||
|
links = set()
|
||||||
|
for raw in URL_ATTR_RE.findall(text) + CSS_URL_RE.findall(text):
|
||||||
|
for candidate in raw.split(','): # srcset
|
||||||
|
candidate = candidate.strip().split(' ')[0]
|
||||||
|
parts = self.local_path(candidate)
|
||||||
|
if not parts:
|
||||||
|
continue
|
||||||
|
path = urllib.parse.unquote(parts.path)
|
||||||
|
if ASSET_EXT_RE.search(path):
|
||||||
|
self.assets.add(path)
|
||||||
|
elif self.follow and not parts.query and parts.netloc in ('', self.dev_host, self.prod_host) \
|
||||||
|
and not path.startswith(('/typo3', AJAX_PATH)):
|
||||||
|
links.add(self.dev + path)
|
||||||
|
return links
|
||||||
|
|
||||||
|
# --- pages and AJAX tables ---------------------------------------------
|
||||||
|
|
||||||
|
def ajax_file(self, url):
|
||||||
|
"""Fetch an AJAX table, store it, return its static URL."""
|
||||||
|
url = html.unescape(url)
|
||||||
|
if url.startswith('/'):
|
||||||
|
url = self.dev + url
|
||||||
|
name = '/_ajax/%s.html' % hashlib.sha1(url.replace(self.dev, '').encode()).hexdigest()
|
||||||
|
with self.lock:
|
||||||
|
name_lock = self.ajax_locks.setdefault(name, threading.RLock())
|
||||||
|
# Every reference waits for the first fetch, so a failed table keeps its original URL everywhere
|
||||||
|
with name_lock:
|
||||||
|
if name not in self.ajax_results:
|
||||||
|
# Set before fetching: a response referencing itself must not recurse.
|
||||||
|
# The JS appends "&pa=..."; the dummy query keeps that harmless
|
||||||
|
self.ajax_results[name] = name + '?s=1'
|
||||||
|
status, body, _ = self.fetch(url)
|
||||||
|
if status != 200:
|
||||||
|
self.error('%s → %s' % (url, status))
|
||||||
|
self.ajax_results[name] = url
|
||||||
|
else:
|
||||||
|
self.write(name, self.to_prod(self.process(self.strip_dynamic(body, ''))))
|
||||||
|
with self.lock:
|
||||||
|
self.stats['ajax'] += 1
|
||||||
|
return self.ajax_results[name]
|
||||||
|
|
||||||
|
def strip_dynamic(self, text, path):
|
||||||
|
"""Swap contact forms for a contact notice and drop the search UI."""
|
||||||
|
phone, tel, email = self.contact
|
||||||
|
lead, body = CONTACT_TEXT['en' if path.startswith('/en/') else 'de']
|
||||||
|
notice = CONTACT_NOTICE.format(lead=lead, text=body, phone=phone, tel=tel, email=email)
|
||||||
|
text, forms = replace_elements(text, 'form', CONTACT_FORM_RE, notice)
|
||||||
|
removed = 0
|
||||||
|
for tag, start_re in (HYBRID_SEARCH_ELEMENTS if self.hybrid else SEARCH_ELEMENTS):
|
||||||
|
text, n = replace_elements(text, tag, start_re, '')
|
||||||
|
removed += n
|
||||||
|
with self.lock:
|
||||||
|
self.stats['forms'] += forms
|
||||||
|
self.stats['search'] += removed
|
||||||
|
return text
|
||||||
|
|
||||||
|
def process(self, text):
|
||||||
|
"""Replace AJAX table URLs (recursively fetching them) and collect assets."""
|
||||||
|
text = self.ajax_attr_re.sub(lambda m: m.group(0) if self.is_live_ajax(m.group(2)) else
|
||||||
|
'%s="%s"' % (m.group(1), html.escape(self.ajax_file(m.group(2)))), text)
|
||||||
|
self.collect(text)
|
||||||
|
return text
|
||||||
|
|
||||||
|
def is_live_ajax(self, url):
|
||||||
|
"""Hybrid --live-ajax: keep the URL when TYPO3 13 serves the action."""
|
||||||
|
params = dict(LIVE_AJAX_PARAM_RE.findall(html.unescape(url)))
|
||||||
|
return self.live_ajax and (params.get('controller'), params.get('action')) in LIVE_AJAX
|
||||||
|
|
||||||
|
def is_dynamic(self, path):
|
||||||
|
"""Hybrid: product pages (with their /detail/ variants) and search pages belong to TYPO3 13."""
|
||||||
|
path = path.rstrip('/') + '/'
|
||||||
|
return (path in self.dynamic or any(path.startswith(p + 'detail/') for p in self.dynamic)
|
||||||
|
or any(path.startswith(p) and path != p for p in self.route_prefixes))
|
||||||
|
|
||||||
|
def safe_page(self, url):
|
||||||
|
try:
|
||||||
|
return self.page(url)
|
||||||
|
except Exception as e: # noqa: BLE001 - one broken page must not stop the export
|
||||||
|
self.error('%s: %s: %s' % (url, type(e).__name__, e))
|
||||||
|
return set()
|
||||||
|
|
||||||
|
def page(self, url):
|
||||||
|
status, body, location = self.fetch(url)
|
||||||
|
if status in (301, 302, 303, 307, 308) and location:
|
||||||
|
location = urllib.parse.urljoin(url, location)
|
||||||
|
with self.lock:
|
||||||
|
self.crawl_redirects.append(rewrite_rule(urllib.parse.urlsplit(url).path, self.to_prod(location), status))
|
||||||
|
internal = location.startswith(self.dev + '/') and '?' not in location
|
||||||
|
return {location} if internal and self.follow_redirects else set()
|
||||||
|
if status != 200:
|
||||||
|
self.error('%s → %s' % (url, status))
|
||||||
|
return set()
|
||||||
|
path = urllib.parse.urlsplit(url).path
|
||||||
|
if self.hybrid and self.is_dynamic(path):
|
||||||
|
# Served by TYPO3 13 through index.php; only its links are followed
|
||||||
|
with self.lock:
|
||||||
|
self.stats['dynamic'] += 1
|
||||||
|
return self.collect(body)
|
||||||
|
if not self.hybrid and SEARCH_PAGE_RE.search(path):
|
||||||
|
# Search does not work statically: send visitors to the homepage instead
|
||||||
|
with self.lock:
|
||||||
|
self.crawl_redirects.append(rewrite_rule(path, self.site['prod'] + '/', 302))
|
||||||
|
return set()
|
||||||
|
# Extensionless paths become directories, so /a and /a/b/ can both exist
|
||||||
|
target = path if re.search(r'\.[a-z0-9]+$', path, re.I) else path.rstrip('/') + '/index.html'
|
||||||
|
body = self.strip_dynamic(body, path)
|
||||||
|
links = self.collect(body)
|
||||||
|
self.write(target, self.to_prod(self.process(body)))
|
||||||
|
with self.lock:
|
||||||
|
self.stats['pages'] += 1
|
||||||
|
return links
|
||||||
|
|
||||||
|
def sitemap(self):
|
||||||
|
"""Export sitemap index and children as plain files, return page URLs."""
|
||||||
|
status, index, _ = self.fetch(self.dev + '/sitemap.xml')
|
||||||
|
if status != 200:
|
||||||
|
print(' note: no sitemap.xml (%s), crawling from / only' % status)
|
||||||
|
return []
|
||||||
|
urls = []
|
||||||
|
for loc in re.findall(r'<loc>([^<]+)</loc>', index):
|
||||||
|
child_url = html.unescape(loc)
|
||||||
|
name = urllib.parse.parse_qs(urllib.parse.urlsplit(child_url).query).get('sitemap', ['pages'])[0]
|
||||||
|
_, child, _ = self.fetch(child_url)
|
||||||
|
urls += [html.unescape(u) for u in re.findall(r'<loc>([^<]+)</loc>', child)]
|
||||||
|
self.write('/sitemap-%s.xml' % name, self.to_prod(child))
|
||||||
|
index = index.replace(loc, self.dev + '/sitemap-%s.xml' % name)
|
||||||
|
self.write('/sitemap.xml', self.to_prod(index))
|
||||||
|
return [u for u in urls if u.startswith(self.dev)]
|
||||||
|
|
||||||
|
def crawl(self, start):
|
||||||
|
queue = list(dict.fromkeys(start))
|
||||||
|
self.seen.update(queue)
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
|
||||||
|
while queue:
|
||||||
|
batch, queue = queue, []
|
||||||
|
for links in pool.map(self.safe_page, batch):
|
||||||
|
for link in links:
|
||||||
|
if link not in self.seen:
|
||||||
|
self.seen.add(link)
|
||||||
|
queue.append(link)
|
||||||
|
|
||||||
|
def redirects(self):
|
||||||
|
"""Backend redirects (sys_redirect) as RewriteRules, targets resolved by TYPO3 itself."""
|
||||||
|
host = urllib.parse.urlsplit(self.site['prod']).netloc
|
||||||
|
where = "deleted=0 AND disabled=0 AND source_host IN ('%s', '*')" % host
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e',
|
||||||
|
'SELECT source_path FROM sys_redirect WHERE %s AND is_regexp=0; '
|
||||||
|
'SELECT COUNT(*) FROM sys_redirect WHERE %s AND is_regexp=1' % (where, where)],
|
||||||
|
capture_output=True, text=True, check=True).stdout.splitlines()
|
||||||
|
except (OSError, subprocess.CalledProcessError) as e:
|
||||||
|
self.error('sys_redirect: %s' % e)
|
||||||
|
return ''
|
||||||
|
paths, regexp = sorted(set(out[:-1])), int(out[-1])
|
||||||
|
skipped = ['regexp redirects (not exported): %d' % regexp] if regexp else []
|
||||||
|
|
||||||
|
def resolve(path):
|
||||||
|
if '?' in path:
|
||||||
|
return path, 0, 'query string in source path'
|
||||||
|
conn = http.client.HTTPConnection('127.0.0.1', 80, timeout=60)
|
||||||
|
try:
|
||||||
|
conn.request('GET', urllib.parse.quote(path, safe="/%:@!$&'()*+,;=-._~"), headers={'Host': host})
|
||||||
|
res = conn.getresponse()
|
||||||
|
res.read()
|
||||||
|
return path, res.status, res.getheader('Location')
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
rules = ['\t# Redirects from the TYPO3 backend (sys_redirect), resolved by TYPO3 at export time']
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(self.workers) as pool:
|
||||||
|
for path, status, location in pool.map(resolve, paths):
|
||||||
|
if status not in (301, 302, 303, 307, 308) or not location:
|
||||||
|
# Mostly dead entries (target page gone) that 404 in production already
|
||||||
|
skipped.append('%s → %s %s' % (path, status, location or ''))
|
||||||
|
continue
|
||||||
|
# Resolve like a browser: TYPO3 sends "https:/path" for relative targets
|
||||||
|
location = urllib.parse.urljoin(self.dev + path, location)
|
||||||
|
if location.rstrip('/') == (self.dev + path).rstrip('/'):
|
||||||
|
# TYPO3 matches "/a" for "/a/" too, so a redirect to the page at "/a/" loops
|
||||||
|
skipped.append('%s → %s (redirects to itself)' % (path, location))
|
||||||
|
continue
|
||||||
|
target = urllib.parse.urlsplit(location)
|
||||||
|
if target.netloc == self.dev_host and ASSET_EXT_RE.search(target.path):
|
||||||
|
self.assets.add(urllib.parse.unquote(target.path))
|
||||||
|
rules.append(rewrite_rule(path, self.to_prod(location), status))
|
||||||
|
self.stats['redirects'] = len(rules) - 1
|
||||||
|
self.stats['redirects_skipped'] = len(skipped)
|
||||||
|
with open(self.out.rstrip('/') + '-redirects-skipped.txt', 'w', encoding='utf-8') as fh:
|
||||||
|
fh.write('\n'.join(skipped) + '\n')
|
||||||
|
return '\n'.join(rules) + '\n'
|
||||||
|
|
||||||
|
def special_files(self):
|
||||||
|
status, body, _ = self.fetch(self.dev + '/robots.txt')
|
||||||
|
if status == 200:
|
||||||
|
self.write('/robots.txt', self.to_prod(body))
|
||||||
|
# TYPO3 error page, served by Apache via ErrorDocument 404
|
||||||
|
status, body, _ = self.fetch(self.dev + '/static-export-not-found/')
|
||||||
|
if status == 404 and body:
|
||||||
|
self.write('/404.html', self.to_prod(self.process(self.strip_dynamic(body, '/'))))
|
||||||
|
|
||||||
|
# --- static assets -----------------------------------------------------
|
||||||
|
|
||||||
|
def copy_assets(self):
|
||||||
|
pending, copied, missing = list(self.assets), set(), []
|
||||||
|
while pending:
|
||||||
|
path = pending.pop()
|
||||||
|
if path in copied:
|
||||||
|
continue
|
||||||
|
copied.add(path)
|
||||||
|
src = os.path.join(PUBLIC, path.lstrip('/'))
|
||||||
|
if not os.path.isfile(src):
|
||||||
|
missing.append(path)
|
||||||
|
continue
|
||||||
|
dst = os.path.join(self.out, path.lstrip('/'))
|
||||||
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
if path.endswith('.css'): # fonts and images referenced from CSS
|
||||||
|
with open(src, encoding='utf-8', errors='replace') as fh:
|
||||||
|
for ref in CSS_URL_RE.findall(fh.read()):
|
||||||
|
if ref.startswith(('data:', 'http', '#')):
|
||||||
|
continue
|
||||||
|
ref = urllib.parse.urlsplit(ref).path
|
||||||
|
pending.append(os.path.normpath(os.path.join(os.path.dirname(path), ref)))
|
||||||
|
return len(copied) - len(missing), missing
|
||||||
|
|
||||||
|
|
||||||
|
STATIC_HEADER = '''### Begin: Static export ###
|
||||||
|
# Generated by .ddev/static-export/export.py from public/.htaccess.
|
||||||
|
# No PHP: TYPO3's index.php catch-all and StaticFileCache rules are removed.
|
||||||
|
DirectoryIndex index.html
|
||||||
|
ErrorDocument 404 /404.html
|
||||||
|
Options -Indexes -MultiViews
|
||||||
|
<IfModule mod_php7.c>
|
||||||
|
php_flag engine off
|
||||||
|
</IfModule>
|
||||||
|
### End: Static export ###
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
# --- TYPO3 backchannel API (/api/hotel, /api/product) ----------------------------------------
|
||||||
|
|
||||||
|
# MyE&P calls the API on this site; ApiMiddleware itself answers on every host
|
||||||
|
API_SITES = {'ep-reisen'}
|
||||||
|
API_CODE_RE = re.compile(r'^[A-Za-z0-9_-]+$')
|
||||||
|
|
||||||
|
|
||||||
|
def mysql_rows(sql):
|
||||||
|
out = subprocess.run(['mysql', '-h', 'db', '-udb', '-pdb', 'db', '-N', '-B', '-e', sql],
|
||||||
|
capture_output=True, text=True, check=True).stdout
|
||||||
|
return [line.split('\t') for line in out.splitlines() if line]
|
||||||
|
|
||||||
|
|
||||||
|
# Records that only exist as URL arguments of one page (route enhancers), not linked anywhere:
|
||||||
|
# (page uid, table, field, pattern the value must match or None)
|
||||||
|
ROUTE_RECORDS = (
|
||||||
|
(1846, 'tx_epproducts_domain_model_product', 'path_segment', None), # /gruppe/{product}
|
||||||
|
(2736, 'tx_epproducts_domain_model_travelinfo', 'path_segment', None), # /reiseinformationen/{name}
|
||||||
|
(2736, 'tx_epproducts_domain_model_travelinfo', 'travel_code', r'[A-Z]+\d{6}'), # /reiseinformationen/{code}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def page_tree_check(site):
|
||||||
|
"""Return a function telling whether a default-language page uid belongs to the site."""
|
||||||
|
parents = {int(uid): int(pid) for uid, pid in mysql_rows('SELECT uid, pid FROM pages WHERE sys_language_uid = 0')}
|
||||||
|
|
||||||
|
def in_site(uid):
|
||||||
|
for _ in range(100):
|
||||||
|
if uid == site['root']:
|
||||||
|
return True
|
||||||
|
if uid not in parents:
|
||||||
|
return False
|
||||||
|
uid = parents[uid]
|
||||||
|
return False
|
||||||
|
return in_site
|
||||||
|
|
||||||
|
|
||||||
|
def route_pages(site):
|
||||||
|
"""Slug (with trailing slash) of each route page of ROUTE_RECORDS on this site, by page uid."""
|
||||||
|
in_site = page_tree_check(site)
|
||||||
|
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
|
||||||
|
pages = {}
|
||||||
|
for uid in sorted({r[0] for r in ROUTE_RECORDS}):
|
||||||
|
rows = mysql_rows('SELECT slug FROM pages WHERE uid = %d AND deleted = 0 AND hidden = 0' % uid)
|
||||||
|
if rows and in_site(uid):
|
||||||
|
pages[uid] = base + rows[0][0].rstrip('/') + '/'
|
||||||
|
return pages
|
||||||
|
|
||||||
|
|
||||||
|
def route_urls(site):
|
||||||
|
"""Start URLs for the records of ROUTE_RECORDS (not reachable through links or the sitemap)."""
|
||||||
|
pages = route_pages(site)
|
||||||
|
urls = set()
|
||||||
|
for uid, table, field, pattern in ROUTE_RECORDS:
|
||||||
|
if uid not in pages:
|
||||||
|
continue
|
||||||
|
for (value,) in mysql_rows("SELECT DISTINCT %s FROM %s WHERE deleted = 0 AND hidden = 0 AND %s <> ''" % (field, table, field)):
|
||||||
|
if pattern is None or re.fullmatch(pattern, value):
|
||||||
|
urls.add(site['dev'].rstrip('/') + pages[uid] + urllib.parse.quote(value) + '/')
|
||||||
|
return sorted(urls)
|
||||||
|
|
||||||
|
|
||||||
|
def dynamic_paths(site):
|
||||||
|
"""Paths (with trailing slash) of the pages TYPO3 13 serves in hybrid mode: pages of this site
|
||||||
|
holding the product detail or search result plugin, in every language of the site."""
|
||||||
|
in_site = page_tree_check(site)
|
||||||
|
rows = mysql_rows(
|
||||||
|
"SELECT DISTINCT p.slug, p.sys_language_uid, IF(p.sys_language_uid = 0, p.uid, p.l10n_parent) "
|
||||||
|
"FROM pages p JOIN tt_content c ON c.pid = p.uid "
|
||||||
|
"WHERE p.deleted = 0 AND p.hidden = 0 AND c.deleted = 0 AND c.hidden = 0 AND c.CType = 'list' "
|
||||||
|
"AND c.list_type IN ('epproducts_product_detail', 'epproducts_searchresult')")
|
||||||
|
base = urllib.parse.urlsplit(site['dev']).path.rstrip('/')
|
||||||
|
paths = set()
|
||||||
|
for slug, language, default_uid in rows:
|
||||||
|
if int(language) in site['languages'] and in_site(int(default_uid)):
|
||||||
|
paths.add(base + site['languages'][int(language)] + slug.rstrip('/') + '/')
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
FRONT_CONTROLLER_STUB = """<?php
|
||||||
|
// Generated by .ddev/static-export/export.py --hybrid: everything not in this static export is
|
||||||
|
// handled by TYPO3 13 (product pages, /gruppe/, travel info, search, /api/*, /ajax/*, 404).
|
||||||
|
// readlink() is not cached, unlike the realpath of a symlinked include path, so right after a
|
||||||
|
// deploy every request already loads the new release (under new paths, so OPcache needs no reset).
|
||||||
|
$current = %r;
|
||||||
|
$release = readlink($current);
|
||||||
|
if ($release === false) {
|
||||||
|
http_response_code(503);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$release = $release[0] === '/' ? $release : dirname($current) . '/' . $release;
|
||||||
|
|
||||||
|
// Public extension resources (/_assets/<hash>/...) come from the release as well: Apache does not
|
||||||
|
// follow a symlink from the docroot into the release on every vhost, PHP does.
|
||||||
|
$path = rawurldecode((string)strtok($_SERVER['REQUEST_URI'] ?? '', '?'));
|
||||||
|
if (str_starts_with($path, '/_assets/')) {
|
||||||
|
$types = [
|
||||||
|
'css' => 'text/css; charset=utf-8', 'js' => 'text/javascript; charset=utf-8', 'json' => 'application/json',
|
||||||
|
'map' => 'application/json', 'svg' => 'image/svg+xml', 'png' => 'image/png', 'jpg' => 'image/jpeg',
|
||||||
|
'jpeg' => 'image/jpeg', 'gif' => 'image/gif', 'webp' => 'image/webp', 'avif' => 'image/avif',
|
||||||
|
'ico' => 'image/x-icon', 'woff' => 'font/woff', 'woff2' => 'font/woff2', 'ttf' => 'font/ttf',
|
||||||
|
'otf' => 'font/otf', 'eot' => 'application/vnd.ms-fontobject',
|
||||||
|
];
|
||||||
|
$file = realpath($release . '/public' . $path);
|
||||||
|
$type = $types[strtolower(pathinfo((string)$file, PATHINFO_EXTENSION))] ?? null;
|
||||||
|
// realpath() resolves the publishing symlinks; anything outside the release is refused
|
||||||
|
if ($file === false || $type === null || !is_file($file) || !str_starts_with($file, realpath($release) . '/')) {
|
||||||
|
http_response_code(404);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
header('Content-Type: ' . $type);
|
||||||
|
header('Content-Length: ' . filesize($file));
|
||||||
|
header('Last-Modified: ' . gmdate('D, d M Y H:i:s', filemtime($file)) . ' GMT');
|
||||||
|
// Content-hashed build files never change under their name (as in the .htaccess for static files).
|
||||||
|
// The Expires header keeps mod_expires from adding a second, conflicting max-age.
|
||||||
|
$maxAge = preg_match('/\\.[0-9a-f]{8}\\.\\w+$/', $file) ? 31536000 : 86400;
|
||||||
|
header('Cache-Control: public, max-age=' . $maxAge . ($maxAge === 31536000 ? ', immutable' : ''));
|
||||||
|
header('Expires: ' . gmdate('D, d M Y H:i:s', time() + $maxAge) . ' GMT');
|
||||||
|
readfile($file);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
require $release . '/public/index.php';
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def cms_api_key():
|
||||||
|
php = '$c = include "%s/public/typo3conf/LocalConfiguration.php"; echo $c["EXTENSIONS"]["ep_products"]["cmsApiKey"] ?? "";' % APP_ROOT
|
||||||
|
key = subprocess.run(['php', '-r', php], capture_output=True, text=True, check=True).stdout.strip()
|
||||||
|
if not API_CODE_RE.match(key):
|
||||||
|
sys.exit('cmsApiKey missing or with characters the .htaccess rules cannot match literally')
|
||||||
|
return key
|
||||||
|
|
||||||
|
|
||||||
|
class ApiExporter:
|
||||||
|
"""Snapshot /api/hotel and /api/product as JSON files, answered by TYPO3 itself.
|
||||||
|
|
||||||
|
TYPO3 data is frozen while nobody edits, so the responses are too. A product+hotel response is
|
||||||
|
the product response with "hotel": <hotel response> appended (ApiMiddleware json-encodes one
|
||||||
|
array), so pair files are assembled from the single responses and a sample is checked against
|
||||||
|
TYPO3. Pairs that were not generated get the response TYPO3 gives for an unknown hotel.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, exp):
|
||||||
|
self.exp = exp
|
||||||
|
self.key = cms_api_key()
|
||||||
|
self.stats = {'hotels': 0, 'products': 0, 'pairs': 0, 'skipped_codes': []}
|
||||||
|
|
||||||
|
def get(self, query):
|
||||||
|
url = self.exp.dev + '/api/' + query
|
||||||
|
for _ in range(3): # the first call of a record can be slow while TYPO3 renders its images
|
||||||
|
status, body, _ = self.exp.fetch(url, {'X-Api-Key': self.key}, log_errors=False)
|
||||||
|
if status:
|
||||||
|
return status, body
|
||||||
|
self.exp.error('api: %s failed 3 times' % query)
|
||||||
|
return status, body
|
||||||
|
|
||||||
|
def responses(self, endpoint, codes):
|
||||||
|
"""code -> body for every code TYPO3 answers with 200."""
|
||||||
|
codes = sorted(set(codes))
|
||||||
|
fetch = lambda c: self.get('%s?%s=%s' % (endpoint, endpoint, urllib.parse.quote(c)))
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(self.exp.workers) as pool:
|
||||||
|
# Warm-up pass: responses change while TYPO3 is still generating resized images
|
||||||
|
list(pool.map(fetch, codes))
|
||||||
|
results = pool.map(fetch, codes)
|
||||||
|
return {c: body for c, (status, body) in zip(codes, results) if status == 200}
|
||||||
|
|
||||||
|
def write(self, path, body):
|
||||||
|
self.exp.write('/_api/' + path, self.exp.to_prod(body))
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
hotel_codes = collections.defaultdict(set)
|
||||||
|
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_hotel WHERE deleted=0 AND code<>''"):
|
||||||
|
hotel_codes[uid].add(code)
|
||||||
|
aliases = mysql_rows("SELECT entity, code FROM tx_epproducts_domain_model_matchcode WHERE deleted=0 AND code<>''")
|
||||||
|
for uid, code in aliases:
|
||||||
|
hotel_codes[uid].add(code)
|
||||||
|
product_codes = collections.defaultdict(set)
|
||||||
|
for uid, code in mysql_rows("SELECT uid, code FROM tx_epproducts_domain_model_product WHERE deleted=0 AND code<>''"):
|
||||||
|
product_codes[uid].add(code)
|
||||||
|
pairs = mysql_rows(
|
||||||
|
"SELECT uid_local, uid_foreign FROM tx_epproducts_product_hotel_mm"
|
||||||
|
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_date WHERE deleted=0"
|
||||||
|
" UNION SELECT product, hotel FROM tx_epproducts_domain_model_hotel_mapping WHERE deleted=0 AND hidden=0")
|
||||||
|
|
||||||
|
def safe(codes):
|
||||||
|
ok = {c for c in codes if API_CODE_RE.match(c)}
|
||||||
|
self.stats['skipped_codes'] += sorted(set(codes) - ok)
|
||||||
|
return ok
|
||||||
|
|
||||||
|
# ApiMiddleware matches matchcode aliases for products too (join on matchcode.entity)
|
||||||
|
hotels = self.responses('hotel', safe({c for cs in hotel_codes.values() for c in cs}))
|
||||||
|
products = self.responses('product', safe({c for cs in product_codes.values() for c in cs} | {c for _, c in aliases}))
|
||||||
|
# An empty code is not "missing" for ApiMiddleware: it matches a record with an empty code
|
||||||
|
for endpoint, found in (('hotel', hotels), ('product', products)):
|
||||||
|
status, body = self.get('%s?%s=' % (endpoint, endpoint))
|
||||||
|
if status == 200:
|
||||||
|
found['@empty'] = body
|
||||||
|
for code, body in hotels.items():
|
||||||
|
self.write('hotel/%s.json' % code, body)
|
||||||
|
for code, body in products.items():
|
||||||
|
self.write('product/%s.json' % code, body)
|
||||||
|
self.write('product/%s/_unknown-hotel.json' % code, body[:-1] + ',"hotel":[]}')
|
||||||
|
# Product response without its closing brace, for pairs assembled by SSI (pair.shtml)
|
||||||
|
self.write('product-part/%s.part' % code, body[:-1])
|
||||||
|
self.exp.write('/_api/pair.shtml', '<!--#include virtual="/_api/product-part/${REDIRECT_API_PRODUCT}.part" -->'
|
||||||
|
',"hotel":<!--#include virtual="/_api/hotel/${REDIRECT_API_HOTEL}.json" -->}')
|
||||||
|
written = set()
|
||||||
|
for product_uid, hotel_uid in pairs:
|
||||||
|
for pc in product_codes.get(product_uid, ()):
|
||||||
|
for hc in hotel_codes.get(hotel_uid, ()):
|
||||||
|
if pc in products and hc in hotels and (pc, hc) not in written:
|
||||||
|
written.add((pc, hc))
|
||||||
|
self.write('product/%s/%s.json' % (pc, hc), products[pc][:-1] + ',"hotel":' + hotels[hc] + '}')
|
||||||
|
self.stats.update(hotels=len(hotels), products=len(products), pairs=len(written))
|
||||||
|
|
||||||
|
# The assembled files must be byte-identical to what TYPO3 answers
|
||||||
|
sample = sorted(written)[::max(1, len(written) // 40)] + [(pc, 'NO-SUCH-HOTEL') for pc in sorted(p for p in products if p != '@empty')[:5]]
|
||||||
|
for pc, hc in sample:
|
||||||
|
status, body = self.get('product?product=%s&hotel=%s' % (pc, urllib.parse.quote(hc)))
|
||||||
|
expected = products[pc][:-1] + (',"hotel":' + hotels[hc] if hc in hotels else ',"hotel":[]') + '}'
|
||||||
|
if status != 200 or body != expected:
|
||||||
|
self.exp.error('api: assembled /api/product?product=%s&hotel=%s differs from TYPO3' % (pc, hc))
|
||||||
|
|
||||||
|
# Error bodies exactly as ApiMiddleware sends them
|
||||||
|
for name, query in (('hotel-400', 'hotel'), ('hotel-404', 'hotel?hotel=NO-SUCH-CODE'),
|
||||||
|
('product-400', 'product'), ('product-404', 'product?product=NO-SUCH-CODE')):
|
||||||
|
self.write('error/%s.json' % name, self.get(query)[1])
|
||||||
|
self.write('error/403.json', self.exp.fetch(self.exp.dev + '/api/hotel?hotel=x')[1])
|
||||||
|
return API_RULES.replace('@KEY@', self.key)
|
||||||
|
|
||||||
|
|
||||||
|
API_HEADER = r'''# TYPO3 backchannel API: error bodies per endpoint, as ApiMiddleware sends them
|
||||||
|
AddCharset utf-8 .json
|
||||||
|
<If "%{REQUEST_URI} == '/api/hotel'">
|
||||||
|
ErrorDocument 400 /_api/error/hotel-400.json
|
||||||
|
ErrorDocument 403 /_api/error/403.json
|
||||||
|
ErrorDocument 404 /_api/error/hotel-404.json
|
||||||
|
</If>
|
||||||
|
<If "%{REQUEST_URI} == '/api/product'">
|
||||||
|
ErrorDocument 400 /_api/error/product-400.json
|
||||||
|
ErrorDocument 403 /_api/error/403.json
|
||||||
|
ErrorDocument 404 /_api/error/product-404.json
|
||||||
|
</If>
|
||||||
|
# Product + hotel pairs that were not pre-generated are assembled by SSI from the single responses
|
||||||
|
<IfModule mod_include.c>
|
||||||
|
<Files "pair.shtml">
|
||||||
|
Options +Includes
|
||||||
|
SetOutputFilter INCLUDES
|
||||||
|
ForceType "application/json; charset=utf-8"
|
||||||
|
</Files>
|
||||||
|
</IfModule>
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
API_RULES = r''' # TYPO3 backchannel API (/api/hotel, /api/product) as static JSON snapshots taken from
|
||||||
|
# TYPO3 at export time; same status codes and bodies as ApiMiddleware
|
||||||
|
RewriteCond %{ENV:REDIRECT_STATUS} ^$
|
||||||
|
RewriteCond %{IS_SUBREQ} false
|
||||||
|
RewriteRule ^_api/ - [R=404,L]
|
||||||
|
RewriteCond %{HTTP:X-Api-Key} !=@KEY@
|
||||||
|
RewriteCond %{QUERY_STRING} !(^|&)key=@KEY@(&|$)
|
||||||
|
RewriteRule ^api/(hotel|product)$ - [R=403,L]
|
||||||
|
# Only file-safe codes are taken over; anything else leaves the variable unset (404 / "hotel": [])
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)hotel=([A-Za-z0-9_-]+)(&|$)
|
||||||
|
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:%2]
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)product=([A-Za-z0-9_-]+)(&|$)
|
||||||
|
RewriteRule ^api/product$ - [E=API_PRODUCT:%2]
|
||||||
|
# An empty code matches a record with an empty code in TYPO3: snapshot file "@empty" (never a real code)
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)hotel=(&|$)
|
||||||
|
RewriteRule ^api/(hotel|product)$ - [E=API_HOTEL:@empty]
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)product=(&|$)
|
||||||
|
RewriteRule ^api/product$ - [E=API_PRODUCT:@empty]
|
||||||
|
RewriteCond %{QUERY_STRING} !(^|&)hotel=
|
||||||
|
RewriteRule ^api/hotel$ - [R=400,L]
|
||||||
|
RewriteCond %{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)$
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
|
||||||
|
RewriteRule ^api/hotel$ _api/hotel/%{ENV:API_HOTEL}.json [L]
|
||||||
|
RewriteRule ^api/hotel$ - [R=404,L]
|
||||||
|
RewriteCond %{QUERY_STRING} !(^|&)product=
|
||||||
|
RewriteRule ^api/product$ - [R=400,L]
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
||||||
|
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json -f
|
||||||
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/%{ENV:API_HOTEL}.json [L]
|
||||||
|
# Pair not pre-generated but both known: assembled by SSI (exactly what TYPO3 returns)
|
||||||
|
<IfModule mod_include.c>
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
||||||
|
RewriteCond %{ENV:API_PRODUCT}/%{ENV:API_HOTEL} ^(@empty|[A-Za-z0-9_-]+)/(@empty|[A-Za-z0-9_-]+)$
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/product-part/%{ENV:API_PRODUCT}.part -f
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/hotel/%{ENV:API_HOTEL}.json -f
|
||||||
|
RewriteRule ^api/product$ _api/pair.shtml [L]
|
||||||
|
</IfModule>
|
||||||
|
# Hotel unknown (or no mod_include): TYPO3 answers with "hotel": []
|
||||||
|
RewriteCond %{QUERY_STRING} (^|&)hotel=
|
||||||
|
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json -f
|
||||||
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}/_unknown-hotel.json [L]
|
||||||
|
RewriteCond %{QUERY_STRING} !(^|&)hotel=
|
||||||
|
RewriteCond %{ENV:API_PRODUCT} ^(@empty|[A-Za-z0-9_-]+)$
|
||||||
|
RewriteCond %{DOCUMENT_ROOT}/_api/product/%{ENV:API_PRODUCT}.json -f
|
||||||
|
RewriteRule ^api/product$ _api/product/%{ENV:API_PRODUCT}.json [L]
|
||||||
|
RewriteRule ^api/product$ - [R=404,L]
|
||||||
|
'''
|
||||||
|
|
||||||
|
REDIRECTS_PLACEHOLDER = '\t# @@REDIRECTS@@\n'
|
||||||
|
|
||||||
|
STATIC_REWRITE = r'''
|
||||||
|
# Static export: reject multipart requests (CVE-2026-109618 / -109619)
|
||||||
|
RewriteCond %{HTTP:Content-Type} multipart [NC]
|
||||||
|
RewriteRule ^ - [F,L]
|
||||||
|
# No PHP is ever served, also not via a symlinked fileadmin
|
||||||
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
||||||
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
||||||
|
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
|
||||||
|
# Search does not work statically: any search page goes to the homepage
|
||||||
|
RewriteRule (^|/)suche/?$ / [R=302,L]
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
HYBRID_HEADER = '''### Begin: Static export (hybrid) ###
|
||||||
|
# Generated by .ddev/static-export/export.py --hybrid from public/.htaccess.
|
||||||
|
# Files of the static export win; anything else goes to TYPO3 13 through index.php.
|
||||||
|
DirectoryIndex index.html
|
||||||
|
ErrorDocument 404 /404.html
|
||||||
|
Options -Indexes -MultiViews
|
||||||
|
### End: Static export (hybrid) ###
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
HYBRID_REWRITE = r'''
|
||||||
|
# Reject multipart requests (CVE-2026-109618 / -109619); the search sends URL-encoded forms
|
||||||
|
RewriteCond %{HTTP:Content-Type} multipart [NC]
|
||||||
|
RewriteRule ^ - [F,L]
|
||||||
|
# No PHP is served directly; index.php only as the internal hand-over to TYPO3 13 below
|
||||||
|
# (/buchung/ is exempt: it redirects to the booking system further down)
|
||||||
|
RewriteCond %{REQUEST_URI} !^/buchung/
|
||||||
|
RewriteCond %{ENV:REDIRECT_STATUS} ^$ [OR]
|
||||||
|
RewriteCond %{REQUEST_URI} !^/index\.php$
|
||||||
|
RewriteRule \.(php\d?|phtml|phar)$ - [F,L,NC]
|
||||||
|
# TYPO3 13 backend and install tool stay closed to the public
|
||||||
|
RewriteRule ^typo3(/|$) - [F,L]
|
||||||
|
'''
|
||||||
|
|
||||||
|
HYBRID_CATCH_ALL = ''' # Not part of the static export (a directory without index.html counts as missing) => TYPO3 13
|
||||||
|
RewriteCond %{REQUEST_FILENAME} !-f
|
||||||
|
RewriteCond %{REQUEST_FILENAME}/index.html !-f
|
||||||
|
RewriteRule ^.*$ %{ENV:CWD}index.php [QSA,L]
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def build_htaccess(hybrid=False):
|
||||||
|
"""TYPO3's .htaccess without the parts that hand requests to PHP (hybrid: with the hand-over)."""
|
||||||
|
with open(os.path.join(PUBLIC, '.htaccess'), encoding='utf-8') as fh:
|
||||||
|
text = fh.read()
|
||||||
|
text, sfc = re.subn(r'[ \t]*### Begin: StaticFileCache \(preparation\).*?### End: StaticFileCache ###\n', '', text, flags=re.S)
|
||||||
|
text, catch_all = re.subn(r'(?:[ \t]*#[^\n]*\n)*(?:[ \t]*RewriteCond %\{REQUEST_FILENAME\} !-[fdl]\n){3}[ \t]*RewriteRule \^\.\*\$ %\{ENV:CWD\}index\.php \[QSA,L\]\n',
|
||||||
|
lambda m: HYBRID_CATCH_ALL if hybrid else '', text)
|
||||||
|
rewrite = HYBRID_REWRITE if hybrid else STATIC_REWRITE
|
||||||
|
text, engine = re.subn(r'(\n[ \t]*RewriteEngine On\n)', r'\1' + (rewrite + REDIRECTS_PLACEHOLDER).replace('\\', '\\\\'), text, count=1)
|
||||||
|
leftover = re.sub(r'#[^\n]*', '', text.replace(HYBRID_CATCH_ALL, '') if hybrid else text)
|
||||||
|
if (sfc, catch_all, engine) != (1, 1, 1) or re.search(r'RewriteRule[^\n]*index\.php', leftover):
|
||||||
|
sys.exit('public/.htaccess changed shape, adjust build_htaccess() (sfc=%d, catch-all=%d, engine=%d)' % (sfc, catch_all, engine))
|
||||||
|
return (HYBRID_HEADER if hybrid else STATIC_HEADER) + text
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument('--site', action='append', help='site identifier from config/sites (repeatable, default: all)')
|
||||||
|
ap.add_argument('--url', action='append', help='export only these URLs (dev URLs) instead of the sitemap')
|
||||||
|
ap.add_argument('--no-follow', action='store_true', help='do not follow internal links beyond the start URLs')
|
||||||
|
ap.add_argument('--out', default=os.path.join(APP_ROOT, 'var', 'static-export'))
|
||||||
|
ap.add_argument('--workers', type=int, default=8)
|
||||||
|
ap.add_argument('--hybrid', action='store_true', help='leave product pages and search to TYPO3 13 (see module doc)')
|
||||||
|
ap.add_argument('--typo3-current', help='hybrid: absolute path of the TYPO3 13 "current" release symlink on the server')
|
||||||
|
ap.add_argument('--live-ajax', action='store_true', help='hybrid: keep AJAX URLs that TYPO3 13 serves instead of pre-fetching them')
|
||||||
|
args = ap.parse_args()
|
||||||
|
if args.hybrid and not args.typo3_current:
|
||||||
|
ap.error('--hybrid needs --typo3-current')
|
||||||
|
sys.stdout.reconfigure(line_buffering=True)
|
||||||
|
|
||||||
|
all_sites = load_sites()
|
||||||
|
selected = {k: v for k, v in all_sites.items() if not args.site or k in args.site}
|
||||||
|
if not selected:
|
||||||
|
sys.exit('No matching site in %s' % SITES_DIR)
|
||||||
|
|
||||||
|
htaccess = build_htaccess(args.hybrid)
|
||||||
|
failed = False
|
||||||
|
for name, site in selected.items():
|
||||||
|
out = os.path.join(args.out, name)
|
||||||
|
shutil.rmtree(out, ignore_errors=True)
|
||||||
|
exp = Exporter(name, site, all_sites, out, follow=not args.no_follow and not args.url, workers=args.workers,
|
||||||
|
hybrid=args.hybrid, live_ajax=args.live_ajax)
|
||||||
|
print('== %s: %s → %s' % (name, site['dev'], site['prod']))
|
||||||
|
if args.url:
|
||||||
|
start = [u for u in args.url if u.startswith(site['dev'])]
|
||||||
|
else:
|
||||||
|
start = [site['dev'] + '/'] + exp.sitemap()
|
||||||
|
if not args.hybrid:
|
||||||
|
# Linked from nowhere (handed out to groups and customers); in hybrid mode TYPO3 13 serves them
|
||||||
|
routes = route_urls(site)
|
||||||
|
start += routes
|
||||||
|
if routes:
|
||||||
|
print(' route URLs (/gruppe/, /reiseinformationen/): %d' % len(routes))
|
||||||
|
exp.special_files()
|
||||||
|
exp.crawl(start)
|
||||||
|
rules = exp.redirects()
|
||||||
|
site_htaccess = htaccess
|
||||||
|
if args.hybrid:
|
||||||
|
# TYPO3 13 behind index.php
|
||||||
|
# /_assets/ is served by the stub too (no symlink: not every vhost follows one)
|
||||||
|
exp.write('/index.php', FRONT_CONTROLLER_STUB % args.typo3_current.rstrip('/'))
|
||||||
|
if name in API_SITES and not args.url and not args.hybrid:
|
||||||
|
api = ApiExporter(exp)
|
||||||
|
rules = api.run() + rules
|
||||||
|
site_htaccess = API_HEADER + htaccess
|
||||||
|
print(' api: %d hotels, %d products, %d product/hotel pairs, codes skipped (not file-safe): %s'
|
||||||
|
% (api.stats['hotels'], api.stats['products'], api.stats['pairs'], ', '.join(api.stats['skipped_codes']) or '-'))
|
||||||
|
if exp.crawl_redirects:
|
||||||
|
rules += '\t# Redirects returned by TYPO3 while crawling\n' + '\n'.join(sorted(exp.crawl_redirects)) + '\n'
|
||||||
|
exp.write('/.htaccess', site_htaccess.replace(REDIRECTS_PLACEHOLDER, '\n' + rules))
|
||||||
|
copied, missing = exp.copy_assets()
|
||||||
|
print(' pages: %d, ajax tables: %d, redirects: %d (%d skipped, see %s-redirects-skipped.txt), assets: %d copied, %d missing'
|
||||||
|
% (exp.stats['pages'], exp.stats['ajax'], exp.stats['redirects'], exp.stats['redirects_skipped'], name, copied, len(missing)))
|
||||||
|
print(' contact forms replaced: %d, search elements removed: %d' % (exp.stats['forms'], exp.stats['search']))
|
||||||
|
if args.hybrid:
|
||||||
|
print(' left to TYPO3 13: %d pages' % exp.stats['dynamic'])
|
||||||
|
for path in sorted(missing)[:20]:
|
||||||
|
print(' missing asset: %s' % path)
|
||||||
|
for msg in exp.errors[:20]:
|
||||||
|
print(' error: %s' % msg)
|
||||||
|
if len(exp.errors) > 20:
|
||||||
|
print(' ... %d more errors' % (len(exp.errors) - 20))
|
||||||
|
failed = failed or bool(exp.errors)
|
||||||
|
sys.exit(1 if failed else 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user