chore: add exporter scripts for static clones
This commit is contained in:
1 parent
2771be2d67
commit
f4f0857870
7 files changed
+1114
No files matched your search
@@ -0,0 +1,30 @@
|
||||
"""Request every exported page and every local URL referenced in pages/fragments via a local Apache."""
|
||||
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures
|
||||
root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099'
|
||||
class NoRedir(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *a): return None
|
||||
op = urllib.request.build_opener(NoRedir)
|
||||
def get(path):
|
||||
try:
|
||||
with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status
|
||||
except urllib.error.HTTPError as e: return e.code
|
||||
except Exception as e: return str(e)
|
||||
pages, refs = [], set()
|
||||
for d, _, files in os.walk(root):
|
||||
for f in files:
|
||||
if not f.endswith('.html'): continue
|
||||
rel = '/' + os.path.relpath(os.path.join(d, f), root)
|
||||
if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel)
|
||||
t = open(os.path.join(d, f), encoding='utf-8').read()
|
||||
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t):
|
||||
for c in html.unescape(v).split(','):
|
||||
c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c)
|
||||
if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'):
|
||||
refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else ''))
|
||||
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
||||
pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs))))
|
||||
bad_p = {k: v for k, v in pr.items() if v != 200}
|
||||
bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)}
|
||||
print('pages: %d, not 200: %d' % (len(pr), len(bad_p)))
|
||||
print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r)))
|
||||
for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))
|
||||
Reference in new issue
Block a user