chore: add exporter scripts for static clones
This commit is contained in:
1 parent
2771be2d67
commit
f4f0857870
7 files changed
+1114
No files matched your search
@@ -0,0 +1,36 @@
|
||||
"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files."""
|
||||
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections
|
||||
root, prod = sys.argv[1], sys.argv[2].rstrip('/')
|
||||
host = urllib.parse.urlsplit(prod).netloc
|
||||
class NoRedir(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *a): return None
|
||||
op = urllib.request.build_opener(NoRedir)
|
||||
def get(url, body=False):
|
||||
try:
|
||||
with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r:
|
||||
return r.status, (r.read().decode('utf-8', 'replace') if body else '')
|
||||
except urllib.error.HTTPError as e: return e.code, ''
|
||||
except Exception as e: return str(e)[:60], ''
|
||||
pages = []
|
||||
for d, _, files in os.walk(root):
|
||||
if '/_ajax' in d: continue
|
||||
for f in files:
|
||||
if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else ''))
|
||||
pages = sorted(set(p.replace('//', '/') for p in pages))
|
||||
with concurrent.futures.ThreadPoolExecutor(12) as ex:
|
||||
res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages)))
|
||||
refs = set()
|
||||
for p, (st, body) in res.items():
|
||||
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body):
|
||||
for c in html.unescape(v).split(','):
|
||||
c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c)
|
||||
if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'):
|
||||
base = 'https://' + (u.netloc or host)
|
||||
refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else ''))
|
||||
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
||||
rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs))))
|
||||
print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values()))))
|
||||
by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items())
|
||||
print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)]
|
||||
bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))]
|
||||
for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u))
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Request every exported page and every local URL referenced in pages/fragments via a local Apache."""
|
||||
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures
|
||||
root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099'
|
||||
class NoRedir(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *a): return None
|
||||
op = urllib.request.build_opener(NoRedir)
|
||||
def get(path):
|
||||
try:
|
||||
with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status
|
||||
except urllib.error.HTTPError as e: return e.code
|
||||
except Exception as e: return str(e)
|
||||
pages, refs = [], set()
|
||||
for d, _, files in os.walk(root):
|
||||
for f in files:
|
||||
if not f.endswith('.html'): continue
|
||||
rel = '/' + os.path.relpath(os.path.join(d, f), root)
|
||||
if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel)
|
||||
t = open(os.path.join(d, f), encoding='utf-8').read()
|
||||
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t):
|
||||
for c in html.unescape(v).split(','):
|
||||
c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c)
|
||||
if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'):
|
||||
refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else ''))
|
||||
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
||||
pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs))))
|
||||
bad_p = {k: v for k, v in pr.items() if v != 200}
|
||||
bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)}
|
||||
print('pages: %d, not 200: %d' % (len(pr), len(bad_p)))
|
||||
print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r)))
|
||||
for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Compare /api/hotel + /api/product of a static export (Apache on 127.0.0.1:8099) with live local TYPO3.
|
||||
|
||||
Usage: compare_api.py <export dir>. Case list: every hotel/product code, sampled pairs, random
|
||||
unlinked pairs, empty/missing params, missing/wrong/query-string key, direct snapshot access.
|
||||
"""
|
||||
import sys, os, random, importlib.util, urllib.request, urllib.error, concurrent.futures
|
||||
spec = importlib.util.spec_from_file_location('export', '/var/www/html/.ddev/static-export/export.py'); m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
sites = m.load_sites(); exp = m.Exporter('ep-reisen', sites['ep-reisen'], sites, '/nonexistent', follow=False, workers=1); key = m.cms_api_key()
|
||||
class N(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *a): return None
|
||||
op = urllib.request.build_opener(N)
|
||||
def call(url, headers):
|
||||
try:
|
||||
with op.open(urllib.request.Request(url, headers=headers), timeout=60) as r: return r.status, r.read().decode(), r.headers.get('Content-Type')
|
||||
except urllib.error.HTTPError as e: return e.code, e.read().decode(), e.headers.get('Content-Type')
|
||||
typo3 = lambda q, h: call('http://127.0.0.1/' + q, dict(h, **{'Host': 'ep-reisen.ddev.site', 'X-Forwarded-Proto': 'https'}))
|
||||
static = lambda q, h: call('http://127.0.0.1:8099/' + q, dict(h, **{'Host': 'www.ep-reisen.de'}))
|
||||
root = sys.argv[1]; K = {'X-Api-Key': key}
|
||||
hotels = sorted(f[:-5] for f in os.listdir(root + '/_api/hotel')); products = sorted(f[:-5] for f in os.listdir(root + '/_api/product') if f.endswith('.json'))
|
||||
pairs = sorted((p, f[:-5]) for p in products if os.path.isdir(root + '/_api/product/' + p) for f in os.listdir(root + '/_api/product/' + p) if not f.startswith('_'))
|
||||
random.seed(1)
|
||||
cases = [('api/hotel?hotel=' + c, K) for c in hotels] + [('api/product?product=' + c, K) for c in products]
|
||||
cases += [('api/product?product=%s&hotel=%s' % pc, K) for pc in random.sample(pairs, 150)]
|
||||
cases += [('api/product?hotel=%s&product=%s' % (h, p), K) for p, h in random.sample(pairs, 10)]
|
||||
cases += [('api/product?product=%s&hotel=%s' % (p, random.choice(hotels)), K) for p in random.sample(products, 60)]
|
||||
cases += [(q, K) for q in ['api/hotel?hotel=_empty', 'api/product?product=_empty', 'api/hotel?hotel=%40empty', 'api/product?product=3VIIN&hotel=@empty', 'api/product?product=3VIIN&hotel=PMVSW%26A', 'api/hotel?hotel=a%20b', 'api/product?product=%40empty&hotel=', 'api/hotel', 'api/product', 'api/hotel?hotel=', 'api/product?product=', 'api/product?product=&hotel=X',
|
||||
'api/hotel?hotel=NOPE', 'api/product?product=NOPE', 'api/product?product=NOPE&hotel=' + hotels[0],
|
||||
'api/product?product=%s&hotel=' % products[0], 'api/product?product=%s&hotel=NOPE' % products[0]]]
|
||||
cases += [('api/hotel?hotel=' + hotels[0], {}), ('api/hotel?hotel=' + hotels[0], {'X-Api-Key': 'wrong'}),
|
||||
('api/hotel?hotel=%s&key=%s' % (hotels[0], key), {}), ('api/product?key=%s&product=%s' % (key, products[0]), {}),
|
||||
('api/product?product=%s&key=wrong' % products[0], {})]
|
||||
def compare(c):
|
||||
q, h = c; a, b = typo3(q, h), static(q, h)
|
||||
return q, (a[0], exp.to_prod(a[1]), a[2]), b
|
||||
bad = []
|
||||
with concurrent.futures.ThreadPoolExecutor(8) as ex:
|
||||
for q, a, b in ex.map(compare, cases):
|
||||
if a != b: bad.append((q, a, b))
|
||||
print('cases: %d, identical: %d, different: %d' % (len(cases), len(cases) - len(bad), len(bad)))
|
||||
for q, a, b in bad[:6]: print(' DIFF', q, '| typo3', a[0], a[2], '| static', b[0], b[2], repr(b[1][:90]))
|
||||
for q in ['_api/hotel/%s.json' % hotels[0], '_api/pair.shtml', '_api/product-part/%s.part' % products[0]]: print(' direct', q, static(q, {})[0])
|
||||
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
/**
|
||||
* Import the same buspro data twice (XML files, MyE&P API) into the local DB and keep each
|
||||
* result as cmp_<source>_<table>. The live tables are backed up first and restored at the end.
|
||||
*/
|
||||
putenv('TYPO3_PATH_ROOT=/var/www/html/public');
|
||||
putenv('TYPO3_PATH_APP=/var/www/html');
|
||||
$classLoader = require '/var/www/html/vendor/autoload.php';
|
||||
\TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::run(0, \TYPO3\CMS\Core\Core\SystemEnvironmentBuilder::REQUESTTYPE_CLI);
|
||||
\TYPO3\CMS\Core\Core\Bootstrap::init($classLoader);
|
||||
|
||||
use EP\EpProducts\Service\ApiImportSource;
|
||||
use EP\EpProducts\Service\DateImportService;
|
||||
use EP\EpProducts\Service\XmlProductReader;
|
||||
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
$db = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionByName('Default');
|
||||
$tables = ['date' => 'tx_epproducts_domain_model_date', 'room' => 'tx_epproducts_domain_model_room', 'product' => 'tx_epproducts_domain_model_product'];
|
||||
|
||||
$snapshot = function (string $prefix) use ($db, $tables) {
|
||||
foreach ($tables as $short => $table) {
|
||||
$db->executeStatement("DROP TABLE IF EXISTS cmp_{$prefix}_{$short}");
|
||||
$db->executeStatement("CREATE TABLE cmp_{$prefix}_{$short} AS SELECT * FROM {$table}");
|
||||
}
|
||||
};
|
||||
$restore = function () use ($db, $tables) {
|
||||
foreach ($tables as $short => $table) {
|
||||
$db->executeStatement("DELETE FROM {$table}");
|
||||
$db->executeStatement("INSERT INTO {$table} SELECT * FROM cmp_bak_{$short}");
|
||||
}
|
||||
};
|
||||
|
||||
$snapshot('bak');
|
||||
try {
|
||||
$reader = new XmlProductReader(GeneralUtility::getFileAbsFileName('fileadmin/xmlexport'));
|
||||
$rows = iterator_to_array($reader, false);
|
||||
$count = GeneralUtility::makeInstance(DateImportService::class)->import($rows);
|
||||
printf("xml: %d rows, %d dates imported, source generated at %s\n", count($rows), $count,
|
||||
$reader->getSourceGeneratedAt() ? $reader->getSourceGeneratedAt()->format('c') : '?');
|
||||
$snapshot('xml');
|
||||
$restore();
|
||||
|
||||
// Bypass the "already imported" baseline: walk the export pages directly
|
||||
$api = GeneralUtility::makeInstance(ApiImportSource::class);
|
||||
$walk = new \ReflectionMethod($api, 'walkPages');
|
||||
$walk->setAccessible(true);
|
||||
$export = $walk->invoke($api);
|
||||
if (!is_array($export)) {
|
||||
throw new \RuntimeException('MyE&P export walk returned ' . var_export($export, true));
|
||||
}
|
||||
[$apiRows, $generatedAt] = $export;
|
||||
$count = GeneralUtility::makeInstance(DateImportService::class)->import($apiRows);
|
||||
printf("api: %d rows, %d dates imported, source generated at %s\n", count($apiRows), $count,
|
||||
$generatedAt ? $generatedAt->format('c') : '?');
|
||||
$snapshot('api');
|
||||
|
||||
// Raw rows of both sources, for a field-by-field comparison independent of the import
|
||||
file_put_contents('/var/www/html/var/rows_xml.json', json_encode($rows));
|
||||
file_put_contents('/var/www/html/var/rows_api.json', json_encode($apiRows));
|
||||
} finally {
|
||||
$restore();
|
||||
echo "live tables restored\n";
|
||||
}
|
||||
Reference in new issue
Block a user