"""Request every exported page and every local URL referenced in pages/fragments via a local Apache.""" import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099' class NoRedir(urllib.request.HTTPRedirectHandler): def redirect_request(self, *a): return None op = urllib.request.build_opener(NoRedir) def get(path): try: with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status except urllib.error.HTTPError as e: return e.code except Exception as e: return str(e) pages, refs = [], set() for d, _, files in os.walk(root): for f in files: if not f.endswith('.html'): continue rel = '/' + os.path.relpath(os.path.join(d, f), root) if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel) t = open(os.path.join(d, f), encoding='utf-8').read() for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t): for c in html.unescape(v).split(','): c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c) if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'): refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else '')) with concurrent.futures.ThreadPoolExecutor(16) as ex: pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs)))) bad_p = {k: v for k, v in pr.items() if v != 200} bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)} print('pages: %d, not 200: %d' % (len(pr), len(bad_p))) print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r))) for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))