chore: add exporter scripts for static clones

This commit is contained in:
fromme committed 2026-10-22 12:29:11 +02:00
1 parent 2771be2d67
commit f4f0857870
7 files changed
+1114

No files matched your search

+36
View File
@@ -0,0 +1,36 @@
"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files."""
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections
root, prod = sys.argv[1], sys.argv[2].rstrip('/')
host = urllib.parse.urlsplit(prod).netloc
class NoRedir(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *a): return None
op = urllib.request.build_opener(NoRedir)
def get(url, body=False):
try:
with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r:
return r.status, (r.read().decode('utf-8', 'replace') if body else '')
except urllib.error.HTTPError as e: return e.code, ''
except Exception as e: return str(e)[:60], ''
pages = []
for d, _, files in os.walk(root):
if '/_ajax' in d: continue
for f in files:
if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else ''))
pages = sorted(set(p.replace('//', '/') for p in pages))
with concurrent.futures.ThreadPoolExecutor(12) as ex:
res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages)))
refs = set()
for p, (st, body) in res.items():
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body):
for c in html.unescape(v).split(','):
c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c)
if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'):
base = 'https://' + (u.netloc or host)
refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else ''))
with concurrent.futures.ThreadPoolExecutor(16) as ex:
rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs))))
print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values()))))
by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items())
print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)]
bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))]
for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u))
+30
View File
@@ -0,0 +1,30 @@
"""Request every exported page and every local URL referenced in pages/fragments via a local Apache."""
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures
root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099'
class NoRedir(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *a): return None
op = urllib.request.build_opener(NoRedir)
def get(path):
try:
with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status
except urllib.error.HTTPError as e: return e.code
except Exception as e: return str(e)
pages, refs = [], set()
for d, _, files in os.walk(root):
for f in files:
if not f.endswith('.html'): continue
rel = '/' + os.path.relpath(os.path.join(d, f), root)
if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel)
t = open(os.path.join(d, f), encoding='utf-8').read()
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t):
for c in html.unescape(v).split(','):
c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c)
if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'):
refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else ''))
with concurrent.futures.ThreadPoolExecutor(16) as ex:
pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs))))
bad_p = {k: v for k, v in pr.items() if v != 200}
bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)}
print('pages: %d, not 200: %d' % (len(pr), len(bad_p)))
print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r)))
for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))
+41
View File
@@ -0,0 +1,41 @@
"""Compare /api/hotel + /api/product of a static export (Apache on 127.0.0.1:8099) with live local TYPO3.
Usage: compare_api.py <export dir>. Case list: every hotel/product code, sampled pairs, random
unlinked pairs, empty/missing params, missing/wrong/query-string key, direct snapshot access.
"""
import sys, os, random, importlib.util, urllib.request, urllib.error, concurrent.futures
spec = importlib.util.spec_from_file_location('export', '/var/www/html/.ddev/static-export/export.py'); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
sites = m.load_sites(); exp = m.Exporter('ep-reisen', sites['ep-reisen'], sites, '/nonexistent', follow=False, workers=1); key = m.cms_api_key()
class N(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *a): return None
op = urllib.request.build_opener(N)
def call(url, headers):
try:
with op.open(urllib.request.Request(url, headers=headers), timeout=60) as r: return r.status, r.read().decode(), r.headers.get('Content-Type')
except urllib.error.HTTPError as e: return e.code, e.read().decode(), e.headers.get('Content-Type')
typo3 = lambda q, h: call('http://127.0.0.1/' + q, dict(h, **{'Host': 'ep-reisen.ddev.site', 'X-Forwarded-Proto': 'https'}))
static = lambda q, h: call('http://127.0.0.1:8099/' + q, dict(h, **{'Host': 'www.ep-reisen.de'}))
root = sys.argv[1]; K = {'X-Api-Key': key}
hotels = sorted(f[:-5] for f in os.listdir(root + '/_api/hotel')); products = sorted(f[:-5] for f in os.listdir(root + '/_api/product') if f.endswith('.json'))
pairs = sorted((p, f[:-5]) for p in products if os.path.isdir(root + '/_api/product/' + p) for f in os.listdir(root + '/_api/product/' + p) if not f.startswith('_'))
random.seed(1)
cases = [('api/hotel?hotel=' + c, K) for c in hotels] + [('api/product?product=' + c, K) for c in products]
cases += [('api/product?product=%s&hotel=%s' % pc, K) for pc in random.sample(pairs, 150)]
cases += [('api/product?hotel=%s&product=%s' % (h, p), K) for p, h in random.sample(pairs, 10)]
cases += [('api/product?product=%s&hotel=%s' % (p, random.choice(hotels)), K) for p in random.sample(products, 60)]
cases += [(q, K) for q in ['api/hotel?hotel=_empty', 'api/product?product=_empty', 'api/hotel?hotel=%40empty', 'api/product?product=3VIIN&hotel=@empty', 'api/product?product=3VIIN&hotel=PMVSW%26A', 'api/hotel?hotel=a%20b', 'api/product?product=%40empty&hotel=', 'api/hotel', 'api/product', 'api/hotel?hotel=', 'api/product?product=', 'api/product?product=&hotel=X',
'api/hotel?hotel=NOPE', 'api/product?product=NOPE', 'api/product?product=NOPE&hotel=' + hotels[0],
'api/product?product=%s&hotel=' % products[0], 'api/product?product=%s&hotel=NOPE' % products[0]]]
cases += [('api/hotel?hotel=' + hotels[0], {}), ('api/hotel?hotel=' + hotels[0], {'X-Api-Key': 'wrong'}),
('api/hotel?hotel=%s&key=%s' % (hotels[0], key), {}), ('api/product?key=%s&product=%s' % (key, products[0]), {}),
('api/product?product=%s&key=wrong' % products[0], {})]
def compare(c):
q, h = c; a, b = typo3(q, h), static(q, h)
return q, (a[0], exp.to_prod(a[1]), a[2]), b
bad = []
with concurrent.futures.ThreadPoolExecutor(8) as ex:
for q, a, b in ex.map(compare, cases):
if a != b: bad.append((q, a, b))
print('cases: %d, identical: %d, different: %d' % (len(cases), len(cases) - len(bad), len(bad)))
for q, a, b in bad[:6]: print(' DIFF', q, '| typo3', a[0], a[2], '| static', b[0], b[2], repr(b[1][:90]))
for q in ['_api/hotel/%s.json' % hotels[0], '_api/pair.shtml', '_api/product-part/%s.part' % products[0]]: print(' direct', q, static(q, {})[0])
@@ -0,0 +1,64 @@
<?php
/**
* Import the same buspro data twice (XML files, MyE&P API) into the local DB and keep each
* result as cmp_<source>_<table>. The live tables are backed up first and restored at the end.
*/
putenv('TYPO3_PATH_ROOT=/var/www/html/public');
putenv('TYPO3_PATH_APP=/var/www/html');
$classLoader = require '/var/www/html/vendor/autoload.php';
\TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(0, \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::REQUESTTYPE_CLI);
\TYPO3\CMS\Core\Core\Bootstrap::init($classLoader);
use EP\EpProducts\Service\ApiImportSource;
use EP\EpProducts\Service\DateImportService;
use EP\EpProducts\Service\XmlProductReader;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Utility\GeneralUtility;
$db = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
$tables = ['date' => 'tx_epproducts_domain_model_date', 'room' => 'tx_epproducts_domain_model_room', 'product' => 'tx_epproducts_domain_model_product'];
$snapshot = function (string $prefix) use ($db, $tables) {
foreach ($tables as $short => $table) {
$db->executeStatement("DROP TABLE IF EXISTS cmp_{$prefix}_{$short}");
$db->executeStatement("CREATE TABLE cmp_{$prefix}_{$short} AS SELECT * FROM {$table}");
}
};
$restore = function () use ($db, $tables) {
foreach ($tables as $short => $table) {
$db->executeStatement("DELETE FROM {$table}");
$db->executeStatement("INSERT INTO {$table} SELECT * FROM cmp_bak_{$short}");
}
};
$snapshot('bak');
try {
$reader = new XmlProductReader(GeneralUtility::getFileAbsFileName('fileadmin/xmlexport'));
$rows = iterator_to_array($reader, false);
$count = GeneralUtility::makeInstance(DateImportService::class)->import($rows);
printf("xml: %d rows, %d dates imported, source generated at %s\n", count($rows), $count,
$reader->getSourceGeneratedAt() ? $reader->getSourceGeneratedAt()->format('c') : '?');
$snapshot('xml');
$restore();
// Bypass the "already imported" baseline: walk the export pages directly
$api = GeneralUtility::makeInstance(ApiImportSource::class);
$walk = new \ReflectionMethod($api, 'walkPages');
$walk->setAccessible(true);
$export = $walk->invoke($api);
if (!is_array($export)) {
throw new \RuntimeException('MyE&P export walk returned ' . var_export($export, true));
}
[$apiRows, $generatedAt] = $export;
$count = GeneralUtility::makeInstance(DateImportService::class)->import($apiRows);
printf("api: %d rows, %d dates imported, source generated at %s\n", count($apiRows), $count,
$generatedAt ? $generatedAt->format('c') : '?');
$snapshot('api');
// Raw rows of both sources, for a field-by-field comparison independent of the import
file_put_contents('/var/www/html/var/rows_xml.json', json_encode($rows));
file_put_contents('/var/www/html/var/rows_api.json', json_encode($apiRows));
} finally {
$restore();
echo "live tables restored\n";
}